From 2e9a5f4e716a7520a7332e9b1b0722f76013deef Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:59:07 +0000 Subject: [PATCH 1/6] feat: Add OpenAI Batch API Instrumentation --- AGENTS.md | 4 + .../openai-v4-auto-hook.span-tree.json | 1355 ++++++++++++--- .../openai-v4-auto-hook.span-tree.txt | 1334 ++++++++++++--- .../openai-v4-latest-auto-hook.span-tree.json | 1355 ++++++++++++--- .../openai-v4-latest-auto-hook.span-tree.txt | 1334 ++++++++++++--- .../openai-v4-latest-wrapped.span-tree.json | 1355 ++++++++++++--- .../openai-v4-latest-wrapped.span-tree.txt | 1334 ++++++++++++--- .../openai-v4-wrapped.span-tree.json | 1355 ++++++++++++--- .../openai-v4-wrapped.span-tree.txt | 1334 ++++++++++++--- .../openai-v5-auto-hook.span-tree.json | 1355 ++++++++++++--- .../openai-v5-auto-hook.span-tree.txt | 1334 ++++++++++++--- .../openai-v5-latest-auto-hook.span-tree.json | 1355 ++++++++++++--- .../openai-v5-latest-auto-hook.span-tree.txt | 1334 ++++++++++++--- .../openai-v5-latest-wrapped.span-tree.json | 1355 ++++++++++++--- .../openai-v5-latest-wrapped.span-tree.txt | 1334 ++++++++++++--- .../openai-v5-wrapped.span-tree.json | 1355 ++++++++++++--- .../openai-v5-wrapped.span-tree.txt | 1334 ++++++++++++--- .../openai-v6-auto-hook.span-tree.json | 1524 ++++++++++++++--- .../openai-v6-auto-hook.span-tree.txt | 1503 +++++++++++++--- .../openai-v6-latest-auto-hook.span-tree.json | 1524 ++++++++++++++--- .../openai-v6-latest-auto-hook.span-tree.txt | 1503 +++++++++++++--- .../openai-v6-latest-wrapped.span-tree.json | 1524 ++++++++++++++--- .../openai-v6-latest-wrapped.span-tree.txt | 1503 +++++++++++++--- .../openai-v6-wrapped.span-tree.json | 1524 ++++++++++++++--- .../openai-v6-wrapped.span-tree.txt | 1503 +++++++++++++--- .../openai-instrumentation/assertions.ts | 367 ++-- .../openai-instrumentation/scenario.impl.mjs | 146 ++ js/src/exports.ts | 2 + js/src/instrumentation/braintrust-plugin.ts | 4 +- .../plugins/openai-batch-constants.ts | 1 + .../openai-batch-instrumentation.test.ts | 722 ++++++++ .../plugins/openai-batch-instrumentation.ts | 1120 ++++++++++++ .../plugins/openai-channels.ts | 23 + .../plugins/openai-plugin.test.ts | 2 +- .../instrumentation/plugins/openai-plugin.ts | 145 +- .../plugins/openai-span-data.ts | 120 ++ js/src/logger.ts | 23 +- js/src/openai-batch-types.ts | 82 + js/src/openai-batch.ts | 65 + js/src/openai-promise-utils.test.ts | 204 +++ js/src/vendor-sdk-types/openai-common.ts | 2 + js/src/wrappers/oai.ts | 74 +- js/src/wrappers/oai_responses.ts | 25 +- js/src/wrappers/openai-promise-utils.ts | 71 +- 44 files changed, 30373 insertions(+), 6449 deletions(-) create mode 100644 js/src/instrumentation/plugins/openai-batch-constants.ts create mode 100644 js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts create mode 100644 js/src/instrumentation/plugins/openai-batch-instrumentation.ts create mode 100644 js/src/instrumentation/plugins/openai-span-data.ts create mode 100644 js/src/openai-batch-types.ts create mode 100644 js/src/openai-batch.ts create mode 100644 js/src/openai-promise-utils.test.ts diff --git a/AGENTS.md b/AGENTS.md index 6338e8830..91b5f76f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,10 @@ mise install # Install toolchain and dependencies pnpm run build # Build all workspace packages (from repo root) ``` +## Public API + +Keep public exports minimal. Generally, export only the requested runtime APIs and do not export types unless explicitly requested. + ## Instrumentation Use the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch. diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json index 49e7bb1d8..7d6ea7f6e 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +892,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +1002,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1112,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1325,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1431,138 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 27, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 73 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt index cbce77cc0..dcb056028 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt @@ -1,312 +1,586 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +589,816 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 27, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 73 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json index 49e7bb1d8..ba468e90c 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +892,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +1002,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1112,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1325,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1431,138 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 19, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 65 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt index cbce77cc0..de6f2dab6 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt @@ -1,312 +1,586 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +589,816 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 19, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 65 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json index 49e7bb1d8..920f620f5 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +884,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +994,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1104,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1317,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1423,146 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 19, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 65 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt index cbce77cc0..c13222807 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt @@ -1,312 +1,578 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +581,824 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 19, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 65 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json index 49e7bb1d8..d7968e46f 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +884,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +994,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1104,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1317,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1423,146 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 27, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 73 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt index cbce77cc0..5c19f6e7b 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt @@ -1,312 +1,578 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +581,824 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 27, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 73 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json index 49e7bb1d8..b5726d1d3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +892,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +1002,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1112,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1325,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1431,138 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 21, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 67 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt index cbce77cc0..93c5f56e6 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt @@ -1,312 +1,586 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +589,816 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 21, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 67 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json index 49e7bb1d8..e71a94318 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +892,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +1002,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1112,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1325,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1431,138 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 45, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 91 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.23.2", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt index cbce77cc0..917e99cc2 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt @@ -1,312 +1,586 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.23.2", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +589,816 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 45, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 91 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json index 49e7bb1d8..82c330f57 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +884,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +994,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1104,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1317,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1423,146 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 45, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 91 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.23.2", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt index cbce77cc0..0d5d65a88 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt @@ -1,312 +1,578 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.23.2", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +581,824 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 45, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 91 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json index 49e7bb1d8..a7acdaec3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +884,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +994,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1104,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1317,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1423,146 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 together gives us 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 together gives us 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 21, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 67 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt index cbce77cc0..9755e3a63 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt @@ -1,312 +1,578 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +581,824 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 together gives us 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 together gives us 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 21, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 67 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json index a9eea3198..5b5261c56 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +731,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +787,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +796,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +818,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +1037,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1147,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1257,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1364,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1470,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1576,126 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 30, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 76 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1707,82 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 220, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 353 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt index cea5718a8..b5864f6a1 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt @@ -1,374 +1,715 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-image-attachment" + │ "operation": "chat-image-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Describe this image in three words or fewer.", + │ "type": "text" + │ }, + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ }, + │ "type": "image_url" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Solid red.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 8516, + │ "time_to_first_token": 0, + │ "tokens": 8519 │ } ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "operation": "chat-pdf-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Summarize this document in one short phrase.", + │ "type": "text" + │ }, + │ { + │ "file": { + │ "file_data": { + │ "content_type": "application/pdf", + │ "filename": "document.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "filename": "document.pdf" + │ }, + │ "type": "file" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "length", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 24, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 236, + │ "time_to_first_token": 0, + │ "tokens": 260 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -377,279 +718,883 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" + │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-parse" + │ "operation": "responses-parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 20 + 4?" │ output: [ │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "type": "output_text" + │ } │ ], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": null, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "description": null, + │ "name": "NumberAnswer", + │ "schema": { + │ "additionalProperties": false, + │ "properties": { + │ "reasoning": { + │ "type": "string" + │ }, + │ "value": { + │ "type": "integer" + │ } + │ }, + │ "required": [ + │ "value", + │ "reasoning" + │ ], + │ "type": "object" + │ }, + │ "strict": true, + │ "type": "json_schema" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 30, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 46, + │ "time_to_first_token": 0, + │ "tokens": 76 │ } └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.compact [llm] input: [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ] output: [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "id": "", "type": "compaction" } ] metadata: { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 220, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 353 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json index a9eea3198..69ed5df8d 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +731,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +787,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +796,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +818,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +1037,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1147,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1257,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1364,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1470,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1576,126 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 17, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 63 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1707,82 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 190, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 323 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.48.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt index cea5718a8..9e8dca063 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt @@ -1,374 +1,715 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.48.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-image-attachment" + │ "operation": "chat-image-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Describe this image in three words or fewer.", + │ "type": "text" + │ }, + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ }, + │ "type": "image_url" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Solid red.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 8516, + │ "time_to_first_token": 0, + │ "tokens": 8519 │ } ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "operation": "chat-pdf-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Summarize this document in one short phrase.", + │ "type": "text" + │ }, + │ { + │ "file": { + │ "file_data": { + │ "content_type": "application/pdf", + │ "filename": "document.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "filename": "document.pdf" + │ }, + │ "type": "file" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "length", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 24, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 236, + │ "time_to_first_token": 0, + │ "tokens": 260 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -377,279 +718,883 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" + │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-parse" + │ "operation": "responses-parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 20 + 4?" │ output: [ │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "type": "output_text" + │ } │ ], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": null, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "description": null, + │ "name": "NumberAnswer", + │ "schema": { + │ "additionalProperties": false, + │ "properties": { + │ "reasoning": { + │ "type": "string" + │ }, + │ "value": { + │ "type": "integer" + │ } + │ }, + │ "required": [ + │ "value", + │ "reasoning" + │ ], + │ "type": "object" + │ }, + │ "strict": true, + │ "type": "json_schema" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 17, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 46, + │ "time_to_first_token": 0, + │ "tokens": 63 │ } └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.compact [llm] input: [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ] output: [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "id": "", "type": "compaction" } ] metadata: { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 190, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 323 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json index a9eea3198..43b2d172d 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +734,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +779,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +788,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +810,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +1029,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1139,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1249,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1356,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1462,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1568,134 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "20 plus 4 equals 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "20 plus 4 equals 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 17, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 63 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1707,82 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 190, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 323 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.48.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt index cea5718a8..4deb2bed8 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt @@ -1,374 +1,707 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.48.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-image-attachment" + │ "operation": "chat-image-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Describe this image in three words or fewer.", + │ "type": "text" + │ }, + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ }, + │ "type": "image_url" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Solid red.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 8516, + │ "time_to_first_token": 0, + │ "tokens": 8519 │ } ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "operation": "chat-pdf-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Summarize this document in one short phrase.", + │ "type": "text" + │ }, + │ { + │ "file": { + │ "file_data": { + │ "content_type": "application/pdf", + │ "filename": "document.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "filename": "document.pdf" + │ }, + │ "type": "file" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "length", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 24, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 236, + │ "time_to_first_token": 0, + │ "tokens": 260 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -377,279 +710,891 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" + │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-parse" + │ "operation": "responses-parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 20 + 4?" │ output: [ │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "parsed": { + │ "reasoning": "20 plus 4 equals 24.", + │ "value": 24 + │ }, + │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "type": "output_text" + │ } │ ], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": null, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_parsed": { + │ "reasoning": "20 plus 4 equals 24.", + │ "value": 24 + │ }, + │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "description": null, + │ "name": "NumberAnswer", + │ "schema": { + │ "additionalProperties": false, + │ "properties": { + │ "reasoning": { + │ "type": "string" + │ }, + │ "value": { + │ "type": "integer" + │ } + │ }, + │ "required": [ + │ "value", + │ "reasoning" + │ ], + │ "type": "object" + │ }, + │ "strict": true, + │ "type": "json_schema" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 17, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 46, + │ "time_to_first_token": 0, + │ "tokens": 63 │ } └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.compact [llm] input: [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ] output: [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "id": "", "type": "compaction" } ] metadata: { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 190, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 323 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json index a9eea3198..034953fc9 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +734,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +779,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +788,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +810,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +1029,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1139,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1249,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1356,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1462,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1568,134 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 30, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 76 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1707,82 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 220, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 353 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt index cea5718a8..6475d02dc 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt @@ -1,374 +1,707 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-image-attachment" + │ "operation": "chat-image-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Describe this image in three words or fewer.", + │ "type": "text" + │ }, + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ }, + │ "type": "image_url" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Solid red.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 8516, + │ "time_to_first_token": 0, + │ "tokens": 8519 │ } ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "operation": "chat-pdf-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Summarize this document in one short phrase.", + │ "type": "text" + │ }, + │ { + │ "file": { + │ "file_data": { + │ "content_type": "application/pdf", + │ "filename": "document.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "filename": "document.pdf" + │ }, + │ "type": "file" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "length", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 24, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 236, + │ "time_to_first_token": 0, + │ "tokens": 260 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -377,279 +710,891 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" + │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-parse" + │ "operation": "responses-parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 20 + 4?" │ output: [ │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "parsed": { + │ "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + │ "value": 24 + │ }, + │ "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "type": "output_text" + │ } │ ], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": null, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_parsed": { + │ "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + │ "value": 24 + │ }, + │ "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "description": null, + │ "name": "NumberAnswer", + │ "schema": { + │ "additionalProperties": false, + │ "properties": { + │ "reasoning": { + │ "type": "string" + │ }, + │ "value": { + │ "type": "integer" + │ } + │ }, + │ "required": [ + │ "value", + │ "reasoning" + │ ], + │ "type": "object" + │ }, + │ "strict": true, + │ "type": "json_schema" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 30, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 46, + │ "time_to_first_token": 0, + │ "tokens": 76 │ } └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.compact [llm] input: [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ] output: [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "id": "", "type": "compaction" } ] metadata: { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 220, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 353 } diff --git a/e2e/scenarios/openai-instrumentation/assertions.ts b/e2e/scenarios/openai-instrumentation/assertions.ts index f9684e4f2..5f7439e41 100644 --- a/e2e/scenarios/openai-instrumentation/assertions.ts +++ b/e2e/scenarios/openai-instrumentation/assertions.ts @@ -2,7 +2,6 @@ import { existsSync } from "node:fs"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; import { beforeAll, describe, expect, test } from "vitest"; -import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { @@ -45,7 +44,9 @@ type RelevantEvent = { type OperationSpec = { childNames: readonly string[]; + nestedChildNames?: readonly string[]; expectsOutput: boolean; + expectsModel?: boolean; expectsTimeToFirstToken: boolean; minOpenAIMajorVersion?: number; name: string; @@ -55,6 +56,12 @@ type OperationSpec = { validate?: (span: CapturedLogEvent | undefined) => void; }; +const EXPECTED_BATCH_OUTPUTS = new Map([ + ["Reply with exactly ALPHA.", "ALPHA"], + ["Reply with exactly BRAVO.", "BRAVO"], + ["Reply with exactly CHARLIE.", undefined], +]); + function asRecord(value: unknown): Record | undefined { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) @@ -233,6 +240,20 @@ const OPERATION_SPECS: readonly OperationSpec[] = [ expect(Array.isArray(span?.output)).toBe(true); }, }, + { + childNames: ["openai.batch"], + nestedChildNames: ["Chat Completion"], + expectsModel: false, + expectsOutput: false, + expectsTimeToFirstToken: false, + name: "openai-batch-operation", + operation: "batch", + testName: "captures resumable OpenAI Batch task and LLM spans", + validate: (span) => { + expect(span?.input).toBeUndefined(); + expect(span?.output).toBeUndefined(); + }, + }, { childNames: ["openai.responses.create"], expectsOutput: true, @@ -320,245 +341,22 @@ function getOperationSpecs(version: string): OperationSpec[] { ); } -function pickMetadata( - metadata: Record | undefined, - keys: string[], -): Json { - if (!metadata) { - return null; - } - - const picked = Object.fromEntries( - keys.flatMap((key) => { - const value = metadata[key]; - if (value === undefined) { - return []; - } - - return [ - [ - key, - key === "openaiSdkVersion" && typeof value === "string" - ? "" - : (value as Json), - ], - ]; - }), - ); - - return Object.keys(picked).length > 0 ? (picked as Json) : null; -} - -function isRecord(value: Json | undefined): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function summarizeMetricPresence(metrics: Json): Json { - if (!isRecord(metrics)) { - return null; - } - - return { - has_time_to_first_token: typeof metrics.time_to_first_token === "number", - } satisfies Json; -} - -function jsonKeysFromText(value: unknown): string[] { - if (typeof value !== "string") { - return []; - } - - try { - const parsed = JSON.parse(value) as unknown; - if ( - typeof parsed !== "object" || - parsed === null || - Array.isArray(parsed) - ) { - return []; - } - - return Object.keys(parsed).sort(); - } catch { - return []; - } -} - -function summarizeInput(input: Json): Json { - if (typeof input === "string") { - return { - kind: "text", - } satisfies Json; - } - - if (!Array.isArray(input)) { - return input === null ? null : ({ kind: typeof input } satisfies Json); - } - - return input.map((message) => { - if (!isRecord(message as Json)) { - return { kind: typeof message } satisfies Json; - } - - return { - content_kind: Array.isArray(message.content) - ? "blocks" - : typeof message.content, - role: message.role ?? null, - } satisfies Json; - }) satisfies Json; -} - -function summarizeChatOutput(output: Json): Json { - if (!Array.isArray(output)) { - return null; - } - - return output.map((choice) => { - if (!isRecord(choice as Json) || !isRecord(choice.message as Json)) { - return null; - } - - return { - json_keys: jsonKeysFromText(choice.message.content), - role: choice.message.role ?? null, - } satisfies Json; - }) satisfies Json; -} - -function summarizeResponsesOutput( - output: Json, - options?: { - // For normalization across SDK versions - dropEmptyOutputTextMessages?: boolean; - }, -): Json { - if (!Array.isArray(output)) { - return null; - } - - const summaries = output.map((item) => { - if (!isRecord(item as Json)) { - return null; - } - - const content = Array.isArray(item.content) ? item.content : []; - const contentTypes = content.flatMap((entry) => - isRecord(entry as Json) && typeof entry.type === "string" - ? [entry.type] - : [], - ); - const jsonKeys = content.flatMap((entry) => - isRecord(entry as Json) ? jsonKeysFromText(entry.text) : [], - ); - - return { - content_types: contentTypes, - json_keys: [...new Set(jsonKeys)].sort(), - role: item.role ?? null, - status: item.status ?? null, - type: item.type ?? null, - } satisfies Json; - }); - - const filtered = options?.dropEmptyOutputTextMessages - ? summaries.filter((item) => { - if (!isRecord(item as Json)) { - return true; - } - - return !( - item.role === "assistant" && - item.status === "completed" && - item.type === "message" && - Array.isArray(item.content_types) && - item.content_types.length === 1 && - item.content_types[0] === "output_text" && - Array.isArray(item.json_keys) && - item.json_keys.length === 0 - ); - }) - : summaries; - - // Deduplicate identical items — the Responses API occasionally returns - // duplicate output entries (e.g., two identical "message" items when - // streaming), which would cause non-deterministic snapshot failures. - const seen = new Set(); - const deduped: Json[] = []; - - for (const summarized of filtered) { - const key = JSON.stringify(summarized); - if (!seen.has(key)) { - seen.add(key); - deduped.push(summarized); +function findOpenAISpan( + events: CapturedLogEvent[], + parentId: string | undefined, + names: readonly string[], +) { + for (const name of names) { + const spans = findChildSpans(events, name, parentId); + if (spans.length > 0) { + return spans.find((span) => span.output !== undefined) ?? spans.at(-1); } } - return deduped; -} - -function summarizeOutput(name: string, output: Json): Json { - if (name === "Chat Completion") { - return summarizeChatOutput(output); - } - - if (name === "Embedding") { - return isRecord(output) - ? ({ - embedding_length: output.embedding_length ?? null, - } satisfies Json) - : null; - } - - if (name === "Moderation") { - return Array.isArray(output) - ? ({ - flagged_count: output.filter( - (entry) => isRecord(entry as Json) && entry.flagged === true, - ).length, - result_count: output.length, - } satisfies Json) - : null; - } - - if ( - name === "openai.responses.create" || - name === "openai.responses.compact" - ) { - return summarizeResponsesOutput(output); - } - - if (name === "openai.responses.parse") { - return summarizeResponsesOutput(output, { - dropEmptyOutputTextMessages: true, - }); - } - - return output === null || output === undefined - ? null - : ({ kind: typeof output } satisfies Json); -} - -function summarizeOpenAIPayload( - event: CapturedLogEvent, - summaryName: string | undefined, -): Json { - const name = summaryName ?? event.span.name ?? ""; - - return { - input: summarizeInput(event.input as Json), - metadata: pickMetadata( - event.row.metadata as Record | undefined, - ["model", "openaiSdkVersion", "operation", "provider", "scenario"], - ), - metrics: summarizeMetricPresence(event.metrics as Json), - name: name || null, - output: summarizeOutput(name, event.output as Json), - type: event.span.type ?? null, - } satisfies Json; + return undefined; } -function findOpenAISpan( +function findOpenAISpans( events: CapturedLogEvent[], parentId: string | undefined, names: readonly string[], @@ -566,11 +364,17 @@ function findOpenAISpan( for (const name of names) { const spans = findChildSpans(events, name, parentId); if (spans.length > 0) { - return spans.find((span) => span.output !== undefined) ?? spans.at(-1); + return spans; } } - return undefined; + return []; +} + +function batchPrompt(span: CapturedLogEvent): string { + const firstMessage = Array.isArray(span.input) ? span.input[0] : undefined; + const content = asRecord(firstMessage)?.content; + return typeof content === "string" ? content : ""; } function buildRelevantEvents( @@ -584,10 +388,27 @@ function buildRelevantEvents( for (const spec of operationSpecs) { const operation = findLatestSpan(events, spec.name)!; relevantEvents.push({ event: operation }); + const providerSpan = findOpenAISpan( + events, + operation.span.id, + spec.childNames, + )!; relevantEvents.push({ - event: findOpenAISpan(events, operation.span.id, spec.childNames)!, + event: providerSpan, summaryName: spec.childNames[0], }); + if (spec.nestedChildNames) { + relevantEvents.push( + ...findOpenAISpans(events, providerSpan.span.id, spec.nestedChildNames) + .sort((left, right) => + batchPrompt(left).localeCompare(batchPrompt(right)), + ) + .map((event) => ({ + event, + summaryName: spec.nestedChildNames?.[0], + })), + ); + } } return relevantEvents; @@ -599,22 +420,13 @@ function buildSpanTree( ): SpanTreeEntry[] { return buildRelevantEvents(events, operationSpecs).map( ({ event, summaryName }) => { - const summary = summarizeOpenAIPayload(event, summaryName) as Record< - string, - Json - >; - const { name: _name, type: _type, ...fields } = summary; - return { event, fields: { - span_attributes: spanTreeFields(event).span_attributes, - ...fields, + ...spanTreeFields(event), + context: event.context, }, - name: - typeof summary.name === "string" - ? summary.name - : (summaryName ?? event.span.name), + name: summaryName ?? event.span.name, }; }, ); @@ -688,6 +500,17 @@ export function defineOpenAIInstrumentationAssertions(options: { spec.childNames, ); expect(spanInstrumentationName(span)).toBe("openai"); + if (spec.nestedChildNames) { + const nested = findOpenAISpans( + events, + span?.span.id, + spec.nestedChildNames, + ); + expect(nested).toHaveLength(EXPECTED_BATCH_OUTPUTS.size); + for (const child of nested) { + expect(spanInstrumentationName(child)).toBe("openai"); + } + } } }); @@ -724,9 +547,12 @@ export function defineOpenAIInstrumentationAssertions(options: { expect(span?.row.metadata).toMatchObject({ provider: "openai", }); - expect( - typeof (span?.row.metadata as { model?: unknown } | undefined)?.model, - ).toBe("string"); + if (spec.expectsModel !== false) { + expect( + typeof (span?.row.metadata as { model?: unknown } | undefined) + ?.model, + ).toBe("string"); + } if (spec.expectsOutput) { expect(span?.output).toBeDefined(); @@ -744,6 +570,41 @@ export function defineOpenAIInstrumentationAssertions(options: { ); } + if (spec.nestedChildNames) { + const nested = findOpenAISpans( + events, + span?.span.id, + spec.nestedChildNames, + ); + expect(nested).toHaveLength(EXPECTED_BATCH_OUTPUTS.size); + expect(span?.span.parentIds).toEqual([operation?.span.id ?? ""]); + expect(nested.map(batchPrompt).sort()).toEqual( + [...EXPECTED_BATCH_OUTPUTS.keys()].sort(), + ); + for (const child of nested) { + const prompt = batchPrompt(child); + const firstChoice = Array.isArray(child.output) + ? asRecord(child.output[0]) + : undefined; + const message = asRecord(firstChoice?.message); + + expect(child.span.parentIds).toEqual([span?.span.id ?? ""]); + expect(child.row.metadata).toMatchObject({ + model: expect.any(String), + provider: "openai", + }); + const expectedOutput = EXPECTED_BATCH_OUTPUTS.get(prompt); + if (expectedOutput) { + expect(message?.content).toBe(expectedOutput); + expect(child.row.error).toBeUndefined(); + } else { + expect(child.output).toBeUndefined(); + expect(child.row.error).toContain("Batch fixture request failed"); + } + expect(child.metrics?.time_to_first_token).toBeUndefined(); + } + } + spec.validate?.(span); }); } diff --git a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs index 076c39492..d17abf4fb 100644 --- a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs @@ -7,6 +7,7 @@ import { runOperation, runTracedScenario, } from "../../helpers/provider-runtime.mjs"; +import { completeOpenAIBatch, createOpenAIBatch } from "braintrust"; const OPENAI_MODEL = "gpt-4o-mini-2024-07-18"; const EMBEDDING_MODEL = "text-embedding-3-small"; @@ -104,6 +105,40 @@ function createMockStreamingClient(options) { : baseClient; } +function createMockBatchClient(options) { + const baseClient = new options.OpenAI({ + apiKey: process.env.OPENAI_API_KEY ?? "test-openai-key", + baseURL: "https://example.test/v1", + fetch: async (_url, init) => { + const params = JSON.parse(String(init?.body)); + return new Response( + JSON.stringify({ + id: "batch_e2e_fixture", + object: "batch", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + completion_window: params.completion_window, + status: "validating", + created_at: 1_740_000_000, + metadata: params.metadata, + request_counts: { completed: 0, failed: 0, total: 0 }, + }), + { + headers: { + "content-type": "application/json", + "x-request-id": "req_batch_e2e_fixture", + }, + status: 200, + }, + ); + }, + }); + + return options.decorateClient + ? options.decorateClient(baseClient) + : baseClient; +} + export async function runOpenAIInstrumentationScenario(options) { const baseClient = new options.OpenAI({ apiKey: process.env.OPENAI_API_KEY, @@ -113,6 +148,7 @@ export async function runOpenAIInstrumentationScenario(options) { ? options.decorateClient(baseClient) : baseClient; const streamFixtureClient = createMockStreamingClient(options); + const batchFixtureClient = createMockBatchClient(options); const openAIMajorVersion = parseMajorVersion(options.openaiSdkVersion); const shouldCheckPrivateFieldMethods = typeof options.decorateClient === "function" && @@ -582,6 +618,116 @@ export async function runOpenAIInstrumentationScenario(options) { }, ); } + + await runOperation("openai-batch-operation", "batch", async () => { + const batchItems = [ + { + customId: "batch_chat_alpha", + prompt: "Reply with exactly ALPHA.", + response: "ALPHA", + }, + { + customId: "batch_chat_bravo", + prompt: "Reply with exactly BRAVO.", + response: "BRAVO", + }, + { + customId: "batch_chat_charlie", + prompt: "Reply with exactly CHARLIE.", + response: "CHARLIE", + }, + ]; + const input = batchItems + .map((item) => + JSON.stringify({ + custom_id: item.customId, + method: "POST", + url: "/v1/chat/completions", + body: { + model: OPENAI_MODEL, + messages: [{ role: "user", content: item.prompt }], + }, + }), + ) + .join("\n"); + const created = await createOpenAIBatch({ + client: batchFixtureClient, + inputFile: { id: "file_batch_e2e_fixture" }, + input, + params: { + completion_window: "24h", + endpoint: "/v1/chat/completions", + }, + }); + + const completedAt = Date.now() / 1000 + 60; + const completed = { + ...created, + status: "completed", + in_progress_at: 1_740_000_010, + completed_at: completedAt, + request_counts: { completed: 2, failed: 1, total: 3 }, + }; + // Start both content requests before awaiting either one. These promises + // model the APIPromise values returned by files.content(). + const outputFile = Promise.resolve( + new Response( + [batchItems[1], batchItems[0]] + .map((item, index) => + JSON.stringify({ + custom_id: item.customId, + response: { + status_code: 200, + body: { + choices: [ + { + index: 0, + finish_reason: "stop", + message: { + role: "assistant", + content: item.response, + }, + }, + ], + usage: { + prompt_tokens: 8 + index, + completion_tokens: 1, + total_tokens: 9 + index, + }, + }, + }, + }), + ) + .join("\n"), + ), + ); + const errorFile = Promise.resolve( + new Response( + JSON.stringify({ + custom_id: batchItems[2].customId, + error: { + code: "fixture_error", + message: "Batch fixture request failed", + }, + }), + ), + ); + await completeOpenAIBatch({ + batch: completed, + input, + // Batch results are not guaranteed to preserve input order. + outputFile, + errorFile, + webhook: { + type: "batch.completed", + created_at: completedAt, + data: { id: completed.id }, + }, + }); + if ((await outputFile).bodyUsed || (await errorFile).bodyUsed) { + throw new Error("Expected batch result responses to remain unread"); + } + }); }, metadata: { openaiSdkVersion: options.openaiSdkVersion, diff --git a/js/src/exports.ts b/js/src/exports.ts index ae4ac3c48..8c0388ac8 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -129,6 +129,8 @@ export { export { registerSandbox } from "./sandbox"; +export { completeOpenAIBatch, createOpenAIBatch } from "./openai-batch"; + // Internal isomorph layer for platform-specific implementations import _internalIso from "./isomorph"; export { _internalIso }; diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index 6bf7af0b6..535276d95 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -393,9 +393,9 @@ export class BraintrustPlugin extends BasePlugin { } } -// Re-export utility functions from OpenAIPlugin for backward compatibility +// Re-export OpenAI instrumentation utilities from their canonical modules. export { parseMetricsFromUsage, - processImagesInOutput, aggregateChatCompletionChunks, } from "./plugins/openai-plugin"; +export { processImagesInOutput } from "./plugins/openai-span-data"; diff --git a/js/src/instrumentation/plugins/openai-batch-constants.ts b/js/src/instrumentation/plugins/openai-batch-constants.ts new file mode 100644 index 000000000..5451014b1 --- /dev/null +++ b/js/src/instrumentation/plugins/openai-batch-constants.ts @@ -0,0 +1 @@ +export const BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY = "braintrust.batch_context"; diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts new file mode 100644 index 000000000..da44afcc6 --- /dev/null +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts @@ -0,0 +1,722 @@ +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + _exportsForTestingOnly, + _internalGetGlobalState, + initLogger, + startSpan, + withCurrent, +} from "../../logger"; +import { configureNode } from "../../node/config"; +import { completeOpenAIBatch, createOpenAIBatch } from "../../openai-batch"; +import type { + OpenAIBatchCreateInputParams, + OpenAIBatchJSONL, + OpenAIFileLike, +} from "../../openai-batch-types"; +import { BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY } from "./openai-batch-constants"; +import { OpenAIPlugin } from "./openai-plugin"; +import { SpanComponentsV4 } from "../../../util/span_identifier_v4"; + +try { + configureNode(); +} catch { + // Best-effort initialization for test environments. +} + +const chatInput = [ + { + custom_id: "ok", + method: "POST", + url: "/v1/chat/completions", + body: { + model: "gpt-test", + messages: [{ role: "user", content: "hi" }], + }, + }, + { + custom_id: "bad", + method: "POST", + url: "/v1/chat/completions", + body: { + model: "gpt-test", + messages: [{ role: "user", content: "fail" }], + }, + }, +]; + +async function createTestBatch(args: { + input: OpenAIBatchJSONL; + inputFile: OpenAIFileLike; + params: OpenAIBatchCreateInputParams; +}) { + return await createOpenAIBatch({ + ...args, + client: { + batches: { + create: async (params) => ({ + id: `batch_${args.inputFile.id}`, + ...params, + status: "validating", + }), + }, + }, + }); +} + +function chunkedJSONLResponse( + records: unknown[], + beforeFirstChunk?: () => Promise, +): Response { + const bytes = new TextEncoder().encode( + records.map((record) => JSON.stringify(record)).join("\n"), + ); + let offset = 0; + return new Response( + new ReadableStream( + { + async pull(controller) { + if (offset === 0) { + await beforeFirstChunk?.(); + } + if (offset < bytes.length) { + controller.enqueue(bytes.slice(offset, offset + 7)); + offset += 7; + } else { + controller.close(); + } + }, + }, + { highWaterMark: 0 }, + ), + ); +} + +describe("OpenAI Batch instrumentation", () => { + let backgroundLogger: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger + >; + let plugin: OpenAIPlugin; + + beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + _internalGetGlobalState()._internalSetTraceContextSigningSecret( + "openai-batch-test-secret", + ); + }); + + beforeEach(() => { + backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ + projectName: "openai-batch-instrumentation.test.ts", + projectId: "test-project-id", + }); + plugin = new OpenAIPlugin(); + plugin.enable(); + }); + + afterEach(() => { + plugin.disable(); + _exportsForTestingOnly.clearTestBackgroundLogger(); + }); + + it("creates a batch and starts deterministic pending spans", async () => { + const inputFile = { id: "file_input", filename: "batch.jsonl" }; + const input = chatInput.map((record) => ({ + ...record, + body: { ...record.body, user: "not-span-metadata" }, + })); + const sourceParams = { + endpoint: "/v1/chat/completions" as const, + completion_window: "24h" as const, + metadata: { user: "value" }, + }; + const params = await createTestBatch({ + inputFile, + input: input.map((record) => JSON.stringify(record)).join("\n"), + params: sourceParams, + }); + + expect(params).toMatchObject({ + input_file_id: "file_input", + endpoint: "/v1/chat/completions", + completion_window: "24h", + metadata: { + user: "value", + [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: expect.any(String), + }, + }); + expect(sourceParams.metadata).toEqual({ user: "value" }); + expect(inputFile).toEqual({ id: "file_input", filename: "batch.jsonl" }); + + const rows = (await backgroundLogger.drain()) as Array>; + expect(rows).toHaveLength(3); + const task = rows.find( + (row) => row.span_attributes?.name === "openai.batch", + ); + const children = rows.filter( + (row) => row.span_attributes?.name === "Chat Completion", + ); + expect(task).toMatchObject({ + metadata: { provider: "openai" }, + span_attributes: { type: "task" }, + }); + expect(task?.input).toBeUndefined(); + expect(task?.metrics).not.toHaveProperty("end"); + expect(task?.span_parents).toEqual([]); + expect(children).toHaveLength(2); + expect(children.map((row) => row.input?.[0]?.content).sort()).toEqual([ + "fail", + "hi", + ]); + expect(children[0]?.metrics).not.toHaveProperty("end"); + expect( + children.find((row) => row.input?.[0]?.content === "hi"), + ).toMatchObject({ + input: [{ role: "user", content: "hi" }], + metadata: { model: "gpt-test", provider: "openai" }, + }); + expect( + children.find((row) => row.input?.[0]?.content === "hi")?.metadata, + ).not.toHaveProperty("user"); + }); + + it("does not start spans when OpenAI returns a different batch context", async () => { + const batch = await createOpenAIBatch({ + inputFile: { id: "file_mismatched_response" }, + input: [chatInput[0]], + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + client: { + batches: { + create: async (params) => ({ + ...params, + id: "batch_mismatched_response", + endpoint: "/v1/responses", + status: "validating", + }), + }, + }, + }); + + expect(batch.endpoint).toBe("/v1/responses"); + expect(await backgroundLogger.drain()).toEqual([]); + }); + + it("keeps separate batch operations distinct when they reuse an input file", async () => { + const args = { + inputFile: { id: "file_reused" }, + input: [chatInput[0]], + params: { + endpoint: "/v1/chat/completions" as const, + completion_window: "24h" as const, + }, + }; + + const first = await createTestBatch(args); + const firstRows = (await backgroundLogger.drain()) as Array< + Record + >; + const second = await createTestBatch(args); + const secondRows = (await backgroundLogger.drain()) as Array< + Record + >; + + expect(first.metadata?.[BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]).not.toBe( + second.metadata?.[BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY], + ); + expect(firstRows).toHaveLength(2); + expect(secondRows).toHaveLength(2); + const firstIds = new Set(firstRows.map((row) => row.id)); + expect(secondRows.some((row) => firstIds.has(row.id))).toBe(false); + }); + + it("signs and verifies batch context with WebCrypto", async () => { + const params = await createTestBatch({ + inputFile: { id: "file_webcrypto" }, + input: [chatInput[0]], + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }); + expect(params.metadata).toHaveProperty(BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY); + await backgroundLogger.drain(); + + await completeOpenAIBatch({ + batch: { + id: "batch_webcrypto", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + status: "completed", + completed_at: 200, + metadata: params.metadata, + }, + input: [chatInput[0]], + outputFile: [ + { + custom_id: "ok", + response: { + status_code: 200, + body: { choices: [], usage: { total_tokens: 0 } }, + }, + }, + ], + }); + expect(await backgroundLogger.drain()).toHaveLength(2); + }); + + it("does not send propagated span event data to OpenAI", async () => { + const parent = startSpan({ + name: "sensitive-parent", + propagatedEvent: { metadata: { secret: "do-not-send" } }, + }); + const batch = await withCurrent(parent, () => + createTestBatch({ + inputFile: { id: "file_minimal_parent" }, + input: [chatInput[0]], + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }), + ); + parent.end(); + + const serialized = JSON.parse( + batch.metadata?.[BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY] ?? "null", + ) as { parent?: unknown; version?: unknown } | null; + expect(serialized?.version).toBe(1); + expect(typeof serialized?.parent).toBe("string"); + const exported = SpanComponentsV4.fromStr(serialized?.parent as string); + expect(exported.data.propagated_event).toBeUndefined(); + }); + + it("handles file promise rejections on early completion paths", async () => { + const batch = { + id: "batch_in_progress", + endpoint: "/v1/chat/completions", + input_file_id: "file_input", + status: "in_progress", + }; + + await expect( + completeOpenAIBatch({ + batch, + input: chatInput, + outputFile: Promise.reject(new Error("output unavailable")), + errorFile: Promise.reject(new Error("errors unavailable")), + }), + ).resolves.toBe(batch); + await new Promise((resolve) => queueMicrotask(resolve)); + expect(await backgroundLogger.drain()).toEqual([]); + }); + + it("completes successful and failed spans from resumable files", async () => { + const params = await createTestBatch({ + inputFile: { id: "file_input" }, + input: chatInput, + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }); + const pendingRows = (await backgroundLogger.drain()) as Array< + Record + >; + const pendingIds = pendingRows.map((row) => row.id).sort(); + const batch = { + id: "batch_complete", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + completion_window: params.completion_window, + status: "completed", + completed_at: 140, + request_counts: { completed: 1, failed: 1, total: 2 }, + metadata: params.metadata, + }; + const outputRecords = [ + { + custom_id: "ok", + response: { + status_code: 200, + body: { + choices: [ + { + index: 0, + message: { role: "assistant", content: "hello 👋" }, + }, + ], + model: "gpt-resolved", + usage: { prompt_tokens: 2, completion_tokens: 3, total_tokens: 5 }, + }, + }, + }, + ]; + const errorRecords = [ + { custom_id: "bad", error: { code: "server_error", message: "failed" } }, + ]; + let readersStarted = 0; + let releaseReaders: (() => void) | undefined; + const bothReadersStarted = new Promise((resolve) => { + releaseReaders = resolve; + }); + const waitForBothReaders = async () => { + readersStarted++; + if (readersStarted === 2) { + releaseReaders?.(); + } + await bothReadersStarted; + }; + const outputResponse = chunkedJSONLResponse( + outputRecords, + waitForBothReaders, + ); + const errorResponse = chunkedJSONLResponse( + errorRecords, + waitForBothReaders, + ); + const completedAt = Date.now() / 1000 + 60; + batch.completed_at = completedAt; + + await expect( + completeOpenAIBatch({ + batch, + input: chatInput, + outputFile: Promise.resolve(outputResponse), + errorFile: Promise.resolve(errorResponse), + }), + ).resolves.toBe(batch); + expect(readersStarted).toBe(2); + expect(outputResponse.bodyUsed).toBe(false); + expect(errorResponse.bodyUsed).toBe(false); + const rows = (await backgroundLogger.drain()) as Array>; + expect(rows).toHaveLength(3); + expect(rows.map((row) => row.id).sort()).toEqual(pendingIds); + const task = rows.find( + (row) => row.span_attributes?.name === "openai.batch", + ); + const children = rows.filter( + (row) => row.span_attributes?.name === "Chat Completion", + ); + expect(task).toMatchObject({ + metrics: { end: completedAt }, + }); + expect(task?.output).toBeUndefined(); + expect(children.find((row) => row.output !== undefined)).toMatchObject({ + metadata: { model: "gpt-resolved", provider: "openai" }, + metrics: { + completion_tokens: 3, + end: completedAt, + prompt_tokens: 2, + tokens: 5, + }, + output: [ + { index: 0, message: { role: "assistant", content: "hello 👋" } }, + ], + }); + expect(children.find((row) => row.error !== undefined)?.error).toContain( + "failed", + ); + + await completeOpenAIBatch({ + batch, + input: chatInput, + outputFile: Promise.resolve(chunkedJSONLResponse(outputRecords)), + errorFile: Promise.resolve(chunkedJSONLResponse(errorRecords)), + }); + const replayRows = (await backgroundLogger.drain()) as Array< + Record + >; + expect(replayRows.map((row) => row.id).sort()).toEqual(pendingIds); + }); + + it("leaves a completed batch pending until all results are available", async () => { + const batch = await createTestBatch({ + inputFile: { id: "file_incomplete" }, + input: chatInput, + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }); + await backgroundLogger.drain(); + + await completeOpenAIBatch({ + batch: { + ...batch, + status: "completed", + completed_at: 400, + request_counts: { completed: 1, failed: 1, total: 2 }, + }, + input: chatInput, + outputFile: [ + { + custom_id: "ok", + response: { + status_code: 200, + body: { choices: [], model: "gpt-resolved" }, + }, + }, + ], + }); + + const rows = (await backgroundLogger.drain()) as Array>; + const task = rows.find( + (row) => row.span_attributes?.name === "openai.batch", + ); + expect(task?.metrics).not.toHaveProperty("end"); + expect(task?.output).toBeUndefined(); + }); + + it("rejects completion input that does not match the signed batch input", async () => { + const params = await createTestBatch({ + inputFile: { id: "file_mismatched" }, + input: chatInput, + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }); + await backgroundLogger.drain(); + + await completeOpenAIBatch({ + batch: { + id: "batch_mismatched", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + status: "completed", + completed_at: 400, + request_counts: { completed: 1, failed: 1, total: 2 }, + metadata: params.metadata, + }, + input: chatInput.map((record) => ({ + ...record, + body: { + ...record.body, + messages: [{ role: "user", content: `changed-${record.custom_id}` }], + }, + })), + outputFile: [ + { + custom_id: "ok", + response: { + status_code: 200, + body: { choices: [], model: "gpt-resolved" }, + }, + }, + ], + errorFile: [{ custom_id: "bad", error: { message: "expected failure" } }], + }); + + expect(await backgroundLogger.drain()).toEqual([]); + }); + + it("falls back from invalid provider terminal timestamps", async () => { + const params = await createTestBatch({ + inputFile: { id: "file_invalid_timestamp" }, + input: [chatInput[0]], + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }); + await backgroundLogger.drain(); + const beforeCompletion = Date.now() / 1000; + + await completeOpenAIBatch({ + batch: { + id: "batch_invalid_timestamp", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + status: "completed", + completed_at: beforeCompletion - 1, + request_counts: { completed: 1, failed: 0, total: 1 }, + metadata: params.metadata, + }, + input: [chatInput[0]], + outputFile: [ + { + custom_id: "ok", + response: { + status_code: 200, + body: { choices: [], model: "gpt-resolved" }, + }, + }, + ], + webhook: { + type: "batch.completed", + created_at: beforeCompletion - 1, + data: { id: "batch_invalid_timestamp" }, + }, + }); + + const rows = (await backgroundLogger.drain()) as Array>; + for (const row of rows) { + expect(row.metrics?.end).toEqual(expect.any(Number)); + expect(row.metrics.end).toBeGreaterThanOrEqual(beforeCompletion); + } + }); + + it("maps Responses records from async input and validates webhook context", async () => { + async function* input() { + yield { + custom_id: "response_1", + method: "POST", + url: "/v1/responses", + body: { model: "gpt-test", input: "hello" }, + }; + } + const params = await createTestBatch({ + inputFile: { id: "file_responses" }, + input: input(), + params: { endpoint: "/v1/responses", completion_window: "24h" }, + }); + await backgroundLogger.drain(); + const batch = { + id: "batch_responses", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + status: "completed", + completed_at: 260, + request_counts: { completed: 1, failed: 0, total: 1 }, + metadata: params.metadata, + }; + const outputFile = [ + { + custom_id: "response_1", + response: { + status_code: 200, + body: { + id: "resp_1", + model: "gpt-test", + output: [{ type: "message", role: "assistant", content: [] }], + usage: { input_tokens: 4, output_tokens: 6, total_tokens: 10 }, + }, + }, + }, + ]; + const completedAt = Date.now() / 1000 + 60; + batch.completed_at = completedAt; + + await completeOpenAIBatch({ + batch, + input: input(), + outputFile, + webhook: { + type: "batch.completed", + created_at: 260, + data: { id: batch.id }, + }, + }); + const rows = (await backgroundLogger.drain()) as Array>; + const responseSpan = rows.find( + (row) => row.span_attributes?.name === "openai.responses.create", + ); + expect(responseSpan).toMatchObject({ + metadata: { + model: "gpt-test", + provider: "openai", + }, + metrics: { + completion_tokens: 6, + end: completedAt, + prompt_tokens: 4, + tokens: 10, + }, + output: [{ type: "message", role: "assistant", content: [] }], + }); + expect(responseSpan?.metrics).not.toHaveProperty("time_to_first_token"); + + await completeOpenAIBatch({ + batch, + input: input(), + outputFile, + webhook: { type: "batch.completed", data: { id: "another_batch" } }, + }); + expect(await backgroundLogger.drain()).toEqual([]); + }); + + it("ends a result span when output processing throws", async () => { + const params = await createTestBatch({ + inputFile: { id: "file_invalid_output" }, + input: [ + { + custom_id: "response_1", + method: "POST", + url: "/v1/responses", + body: { model: "gpt-test", input: "hello" }, + }, + ], + params: { endpoint: "/v1/responses", completion_window: "24h" }, + }); + await backgroundLogger.drain(); + + const output = new Proxy([], { + get(target, property, receiver) { + if (property === "map") { + throw new Error("could not process output"); + } + return Reflect.get(target, property, receiver); + }, + }); + const completedAt = Date.now() / 1000 + 60; + await completeOpenAIBatch({ + batch: { + id: "batch_invalid_output", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + status: "completed", + completed_at: completedAt, + metadata: params.metadata, + }, + input: [ + { + custom_id: "response_1", + method: "POST", + url: "/v1/responses", + body: { model: "gpt-test", input: "hello" }, + }, + ], + outputFile: [ + { + custom_id: "response_1", + response: { + status_code: 200, + body: { output }, + }, + }, + ], + }); + + const rows = (await backgroundLogger.drain()) as Array>; + const child = rows.find( + (row) => row.span_attributes?.name === "openai.responses.create", + ); + expect(child?.error).toContain("could not process output"); + expect(child?.metrics?.end).toBe(completedAt); + }); + + it("leaves conflicting reserved metadata untouched", async () => { + let inputReads = 0; + async function* input() { + inputReads++; + yield* chatInput; + } + const params = await createTestBatch({ + inputFile: { id: "file_input" }, + input: input(), + params: { + endpoint: "/v1/responses", + completion_window: "24h", + metadata: { [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: "user-owned" }, + }, + }); + expect(params.metadata).toEqual({ + [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: "user-owned", + }); + expect(inputReads).toBe(0); + expect(await backgroundLogger.drain()).toEqual([]); + }); +}); diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts new file mode 100644 index 000000000..fcd81e434 --- /dev/null +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts @@ -0,0 +1,1120 @@ +import { debugLogger } from "../../debug-logger"; +import { + _internalGetGlobalState, + _internalStartSpanWithInitialMerge, + _internalStartSpanWithInitialMergeAndParentSpanIds, + getSpanParentObject, + NOOP_SPAN, + type Span, + withCurrent, +} from "../../logger"; +import { parseMetricsFromUsage } from "../../openai-utils"; +import { + INSTRUMENTATION_NAMES, + withSpanInstrumentationName, +} from "../../span-origin"; +import { getCurrentUnixTimestamp } from "../../util"; +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { + SpanComponentsV4, + type SpanComponentsV4Data, +} from "../../../util/span_identifier_v4"; +import type { + CompleteOpenAIBatchArgs, + CreateOpenAIBatchArgs, + OpenAIBatchCreateParams, + OpenAIBatchJSONL, + OpenAIBatchLike, +} from "../../openai-batch-types"; +import { BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY } from "./openai-batch-constants"; +import { openAIChannels } from "./openai-channels"; +import { + extractOpenAIBatchInput, + processImagesInOutput, +} from "./openai-span-data"; + +const SUPPORTED_ENDPOINTS = new Set(["/v1/chat/completions", "/v1/responses"]); +const TERMINAL_STATUSES = new Set([ + "completed", + "failed", + "expired", + "cancelled", +]); +const INPUT_DIGEST_CHUNK_SIZE = 1024 * 1024; + +type BatchContext = { + version: 1; + parent: string; + inputFileId: string; + inputDigest: string; + endpoint: string; + startTime: number; + operationNonce: string; +}; + +type SerializedBatchContext = Omit & { + signature: string; +}; + +type BatchResultRecord = { + value: Record; + source: "output" | "error"; +}; + +type BatchInputRecord = { + customId: string; + spanData?: ReturnType; +}; + +type OpenAIBatchResultFile = + CompleteOpenAIBatchArgs["outputFile"]; + +type PreparedBatch = { + context: BatchContext; + inputs: Map; + params: OpenAIBatchCreateParams; +}; + +function read(value: unknown, key: PropertyKey): unknown { + if (!isObject(value)) { + return undefined; + } + try { + return Reflect.get(value, key); + } catch { + return undefined; + } +} + +function isBatchRecordIterable( + value: unknown, +): value is Iterable | AsyncIterable { + if ( + value === null || + (typeof value !== "object" && typeof value !== "function") + ) { + return false; + } + try { + return ( + typeof Reflect.get(value, Symbol.iterator) === "function" || + typeof Reflect.get(value, Symbol.asyncIterator) === "function" + ); + } catch { + return false; + } +} + +function validCustomId(value: unknown): value is string { + return typeof value === "string" && value.length > 0 && value.length <= 64; +} + +function logBatchInstrumentationError(context: string, error: unknown): void { + debugLogger.debug(`OpenAI Batch instrumentation ${context}:`, error); +} + +function batchContextPayload(context: BatchContext): string { + return JSON.stringify([BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY, context]); +} + +async function importBatchSigningKey(secret: string) { + return await globalThis.crypto.subtle.importKey( + "raw", + new TextEncoder().encode(secret), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign", "verify"], + ); +} + +async function signBatchContext( + context: BatchContext, + secret: string, +): Promise { + try { + const signature = await globalThis.crypto.subtle.sign( + "HMAC", + await importBatchSigningKey(secret), + new TextEncoder().encode(batchContextPayload(context)), + ); + return digestHex(new Uint8Array(signature), 32); + } catch { + return undefined; + } +} + +function signatureBytes(value: string): Uint8Array | undefined { + if (!/^[0-9a-f]{64}$/.test(value)) { + return undefined; + } + return Uint8Array.from(value.match(/.{2}/g) ?? [], (byte) => + Number.parseInt(byte, 16), + ); +} + +async function verifyBatchContext( + context: BatchContext, + signature: string, +): Promise { + const secret = + _internalGetGlobalState()._internalGetTraceContextSigningSecret(); + if (!secret) { + return false; + } + const bytes = signatureBytes(signature); + if (!bytes) { + return false; + } + try { + return await globalThis.crypto.subtle.verify( + "HMAC", + await importBatchSigningKey(secret), + bytes, + new TextEncoder().encode(batchContextPayload(context)), + ); + } catch { + return false; + } +} + +async function parseBatchContext( + value: unknown, + inputFileId: string, + endpoint: string, +): Promise { + if (typeof value !== "string") { + return undefined; + } + + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + return undefined; + } + if ( + !isObject(parsed) || + parsed.version !== 1 || + typeof parsed.parent !== "string" || + typeof parsed.inputDigest !== "string" || + !/^[0-9a-f]{64}$/.test(parsed.inputDigest) || + typeof parsed.startTime !== "number" || + !Number.isFinite(parsed.startTime) || + typeof parsed.operationNonce !== "string" || + !/^[0-9a-f]{32}$/.test(parsed.operationNonce) || + typeof parsed.signature !== "string" + ) { + return undefined; + } + + const context: BatchContext = { + version: 1, + parent: parsed.parent, + inputFileId, + inputDigest: parsed.inputDigest, + endpoint, + startTime: parsed.startTime, + operationNonce: parsed.operationNonce, + }; + if (!(await verifyBatchContext(context, parsed.signature))) { + return undefined; + } + + try { + SpanComponentsV4.fromStr(context.parent); + } catch { + return undefined; + } + return context; +} + +async function exportParent(parent: ReturnType) { + if ("toStr" in parent && typeof parent.toStr === "function") { + return parent.toStr(); + } + return await parent.export(); +} + +async function exportMinimalParent( + parent: ReturnType, +): Promise { + const exported = await exportParent(parent); + if (!exported) { + return undefined; + } + const parsed = SpanComponentsV4.fromStr(exported).data; + const projectId = read(parsed.compute_object_metadata_args, "project_id"); + const projectName = read(parsed.compute_object_metadata_args, "project_name"); + const computeObjectMetadataArgs = + typeof projectId === "string" || typeof projectName === "string" + ? { + ...(typeof projectId === "string" ? { project_id: projectId } : {}), + ...(typeof projectName === "string" + ? { project_name: projectName } + : {}), + } + : undefined; + if (!parsed.object_id && !computeObjectMetadataArgs) { + return undefined; + } + + const routing = parsed.object_id + ? { object_id: parsed.object_id } + : { compute_object_metadata_args: computeObjectMetadataArgs ?? {} }; + // SpanComponents requires all three span identifiers together. The + // conditional below preserves that invariant, but TypeScript cannot infer + // it through the object spreads. + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return new SpanComponentsV4({ + object_type: parsed.object_type, + ...routing, + ...(parsed.row_id && parsed.span_id && parsed.root_span_id + ? { + row_id: parsed.row_id, + span_id: parsed.span_id, + root_span_id: parsed.root_span_id, + } + : {}), + } as SpanComponentsV4Data).toStr(); +} + +async function prepareCreateParams( + args: CreateOpenAIBatchArgs, + parent: ReturnType, + startTime: number, +): Promise { + const params: OpenAIBatchCreateParams = { + ...args.params, + input_file_id: args.inputFile.id, + }; + const endpoint = read(params, "endpoint"); + const inputFileId = read(params, "input_file_id"); + if ( + typeof endpoint !== "string" || + !SUPPORTED_ENDPOINTS.has(endpoint) || + typeof inputFileId !== "string" || + inputFileId.length === 0 + ) { + return undefined; + } + + const metadataValue = read(params, "metadata"); + if ( + metadataValue !== undefined && + metadataValue !== null && + (!isObject(metadataValue) || Array.isArray(metadataValue)) + ) { + logBatchInstrumentationError( + "skipped context injection", + "invalid metadata", + ); + return undefined; + } + const metadata = metadataValue ?? {}; + if ( + Object.prototype.hasOwnProperty.call( + metadata, + BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY, + ) + ) { + logBatchInstrumentationError( + "skipped context injection", + "conflicting reserved metadata", + ); + return undefined; + } + if (Object.keys(metadata).length >= 16) { + logBatchInstrumentationError( + "skipped context injection", + "metadata is full", + ); + return undefined; + } + + const exportedParent = await exportMinimalParent(parent); + if (!exportedParent) { + logBatchInstrumentationError( + "skipped context injection", + "no routable Braintrust parent", + ); + return undefined; + } + const signingSecret = + _internalGetGlobalState()._internalGetTraceContextSigningSecret(); + if (!signingSecret) { + logBatchInstrumentationError( + "skipped context injection", + "no signing secret is available", + ); + return undefined; + } + const inputData = await readBatchInputs(args.input, endpoint); + if (inputData.issues.length > 0) { + logBatchInstrumentationError( + "skipped invalid input file", + inputData.issues[0], + ); + return undefined; + } + const context: BatchContext = { + version: 1, + parent: exportedParent, + inputFileId, + inputDigest: inputData.inputDigest, + endpoint, + startTime, + operationNonce: randomBatchNonce(), + }; + const signature = await signBatchContext(context, signingSecret); + if (!signature) { + logBatchInstrumentationError( + "skipped context injection", + "could not sign batch context", + ); + return undefined; + } + const serialized = JSON.stringify({ + version: context.version, + parent: context.parent, + inputDigest: context.inputDigest, + startTime: context.startTime, + operationNonce: context.operationNonce, + signature, + } satisfies SerializedBatchContext); + if (serialized.length > 512) { + logBatchInstrumentationError( + "skipped context injection", + "signed context exceeds OpenAI metadata limits", + ); + return undefined; + } + + return { + context, + inputs: inputData.inputs, + params: { + ...params, + metadata: { + ...metadata, + [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: serialized, + }, + }, + }; +} + +async function deterministicDigest(namespace: string, ...parts: string[]) { + const encoded = new TextEncoder().encode( + [namespace, ...parts].map((part) => `${part.length}:${part}`).join("\0"), + ); + return new Uint8Array( + await globalThis.crypto.subtle.digest("SHA-256", encoded), + ); +} + +function digestHex(bytes: Uint8Array, length: number): string { + return Array.from(bytes.slice(0, length)) + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +function digestUuid(bytes: Uint8Array): string { + const hex = digestHex(bytes, 16); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} + +function randomBatchNonce(): string { + return digestHex(globalThis.crypto.getRandomValues(new Uint8Array(16)), 16); +} + +async function batchSpanIds(operationNonce: string): Promise<{ + rowId: string; + spanId: string; + rootSpanId: string; +}> { + const [row, span, root] = await Promise.all([ + deterministicDigest("openai:batch:row", operationNonce), + deterministicDigest("openai:batch:span", operationNonce), + deterministicDigest("openai:batch:root", operationNonce), + ]); + return { + rowId: digestUuid(row), + spanId: digestHex(span, 8), + rootSpanId: digestHex(root, 16), + }; +} + +async function childSpanIds(operationNonce: string, customId: string) { + const [row, span] = await Promise.all([ + deterministicDigest("openai:batch:child:row", operationNonce, customId), + deterministicDigest("openai:batch:child:span", operationNonce, customId), + ]); + return { rowId: digestUuid(row), spanId: digestHex(span, 8) }; +} + +async function batchContextFromBatch( + batch: OpenAIBatchLike, +): Promise { + const metadata = read(batch, "metadata"); + const inputFileId = read(batch, "input_file_id"); + const endpoint = read(batch, "endpoint"); + if ( + !isObject(metadata) || + typeof inputFileId !== "string" || + typeof endpoint !== "string" + ) { + return undefined; + } + return await parseBatchContext( + read(metadata, BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY), + inputFileId, + endpoint, + ); +} + +async function startBatchSpan(context: BatchContext): Promise { + const ids = await batchSpanIds(context.operationNonce); + const parent = SpanComponentsV4.fromStr(context.parent); + const hasParentSpan = Boolean( + parent.data.row_id && parent.data.span_id && parent.data.root_span_id, + ); + return withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMergeAndParentSpanIds( + withSpanInstrumentationName( + { + name: "openai.batch", + type: SpanTypeAttribute.TASK, + parent: context.parent, + ...(!hasParentSpan + ? { + parentSpanIds: { + parentSpanIds: [], + rootSpanId: ids.rootSpanId, + }, + } + : {}), + spanId: ids.spanId, + startTime: context.startTime, + event: { + id: ids.rowId, + metadata: { provider: "openai" }, + }, + }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ), + ); +} + +async function startBatchChild( + context: BatchContext, + taskParent: string, + input: BatchInputRecord, +): Promise { + const ids = await childSpanIds(context.operationNonce, input.customId); + return withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMerge( + withSpanInstrumentationName( + { + name: + context.endpoint === "/v1/chat/completions" + ? "Chat Completion" + : "openai.responses.create", + type: SpanTypeAttribute.LLM, + parent: taskParent, + spanId: ids.spanId, + startTime: context.startTime, + event: { + id: ids.rowId, + ...(input.spanData?.input !== undefined + ? { input: input.spanData.input } + : {}), + metadata: { + ...input.spanData?.metadata, + provider: "openai", + }, + }, + }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ), + ); +} + +async function* jsonlRecords( + file: OpenAIBatchJSONL | OpenAIBatchResultFile, + onIssue: (error: Error) => void = () => {}, +): AsyncGenerator { + const resolvedFile = await file; + if (resolvedFile === undefined) { + return; + } + if (typeof resolvedFile === "string") { + for (const line of resolvedFile.split(/\r?\n/)) { + if (!line.trim()) { + continue; + } + try { + yield JSON.parse(line); + } catch (error) { + logBatchInstrumentationError("skipped malformed JSONL", error); + onIssue(new Error("OpenAI Batch file contains malformed JSONL")); + } + } + return; + } + + const clone = read(resolvedFile, "clone"); + let response = resolvedFile; + if (typeof clone === "function") { + try { + response = Reflect.apply(clone, resolvedFile, []); + } catch (error) { + logBatchInstrumentationError("could not clone JSONL response", error); + onIssue(new Error("OpenAI Batch response could not be cloned")); + return; + } + } + const body = read(response, "body"); + const getReader = read(body, "getReader"); + if (isObject(body) && typeof getReader === "function") { + let reader: unknown; + try { + reader = Reflect.apply(getReader, body, []); + if (!isObject(reader)) { + throw new Error("Response body returned an invalid stream reader"); + } + + const decoder = new TextDecoder(); + let pending = ""; + while (true) { + const readChunk = read(reader, "read"); + if (typeof readChunk !== "function") { + throw new Error("Response body stream reader has no read method"); + } + const chunk = await Reflect.apply(readChunk, reader, []); + if (!isObject(chunk)) { + throw new Error("Response body stream returned an invalid chunk"); + } + if (chunk.done === true) { + pending += decoder.decode(); + break; + } + if (!(chunk.value instanceof Uint8Array)) { + throw new Error("Response body stream returned a non-byte chunk"); + } + + pending += decoder.decode(chunk.value, { stream: true }); + let newline = pending.indexOf("\n"); + while (newline !== -1) { + const line = pending.slice(0, newline).replace(/\r$/, ""); + pending = pending.slice(newline + 1); + if (line.trim()) { + try { + yield JSON.parse(line); + } catch (error) { + logBatchInstrumentationError("skipped malformed JSONL", error); + onIssue( + new Error("OpenAI Batch response contains malformed JSONL"), + ); + } + } + newline = pending.indexOf("\n"); + } + } + + if (pending.trim()) { + try { + yield JSON.parse(pending.replace(/\r$/, "")); + } catch (error) { + logBatchInstrumentationError("skipped malformed JSONL", error); + onIssue(new Error("OpenAI Batch response contains malformed JSONL")); + } + } + } catch (error) { + logBatchInstrumentationError("could not read JSONL response body", error); + onIssue(new Error("OpenAI Batch response body could not be read")); + } finally { + const releaseLock = read(reader, "releaseLock"); + if (isObject(reader) && typeof releaseLock === "function") { + try { + Reflect.apply(releaseLock, reader, []); + } catch (error) { + logBatchInstrumentationError( + "could not release stream reader", + error, + ); + } + } + } + return; + } + + if (isBatchRecordIterable(resolvedFile)) { + for await (const record of resolvedFile) { + yield record; + } + } else { + logBatchInstrumentationError("skipped invalid JSONL source", resolvedFile); + onIssue(new Error("OpenAI Batch file source is invalid")); + } +} + +async function startPreparedBatch( + inputs: Map, + context: BatchContext, +): Promise { + const task = await startBatchSpan(context); + const taskParent = await task.export(); + for (const input of inputs.values()) { + try { + await startBatchChild(context, taskParent, input); + } catch (error) { + logBatchInstrumentationError("skipped invalid input record", error); + } + } +} + +export const interceptOpenAIBatchCreate: Parameters< + typeof openAIChannels.batchesCreate.intercept +>[0] = async (target, thisArg, args) => { + const parent = getSpanParentObject(); + const startTime = getCurrentUnixTimestamp(); + let prepared: PreparedBatch | undefined; + try { + prepared = await prepareCreateParams(args[0], parent, startTime); + } catch (error) { + logBatchInstrumentationError("could not prepare batch", error); + } + + const batch = await Reflect.apply(target, thisArg, [ + prepared + ? { + ...args[0], + params: prepared.params, + } + : args[0], + ]); + if (prepared) { + try { + const returnedContext = await batchContextFromBatch(batch); + if ( + !returnedContext || + batchContextPayload(returnedContext) !== + batchContextPayload(prepared.context) + ) { + logBatchInstrumentationError( + "did not start batch spans", + "returned batch context does not match the submitted batch", + ); + } else { + await startPreparedBatch(prepared.inputs, prepared.context); + } + } catch (error) { + logBatchInstrumentationError("could not start batch spans", error); + } + } + return batch; +}; + +function errorFromResult(result: BatchResultRecord): Error | undefined { + const error = read(result.value, "error"); + if (isObject(error)) { + const message = read(error, "message"); + return new Error( + typeof message === "string" ? message : "OpenAI Batch request failed", + ); + } + const response = read(result.value, "response"); + const statusCode = read(response, "status_code"); + if ( + result.source === "error" || + (typeof statusCode === "number" && (statusCode < 200 || statusCode >= 300)) + ) { + const body = read(response, "body"); + const bodyError = read(body, "error"); + const message = read(bodyError, "message"); + return new Error( + typeof message === "string" ? message : "OpenAI Batch request failed", + ); + } + return undefined; +} + +async function completeBatchResult( + context: BatchContext, + taskParent: string, + endTime: number, + input: BatchInputRecord, + result: BatchResultRecord, +): Promise { + const child = await startBatchChild(context, taskParent, input); + try { + const resultError = errorFromResult(result); + const response = read(result.value, "response"); + const responseBody = read(response, "body"); + if (resultError) { + child.log({ error: resultError }); + } else if (isObject(responseBody)) { + if (context.endpoint === "/v1/chat/completions") { + const model = read(responseBody, "model"); + child.log({ + output: read(responseBody, "choices"), + ...(typeof model === "string" ? { metadata: { model } } : {}), + metrics: parseMetricsFromUsage(read(responseBody, "usage")), + }); + } else { + const model = read(responseBody, "model"); + child.log({ + output: processImagesInOutput(read(responseBody, "output")), + ...(typeof model === "string" ? { metadata: { model } } : {}), + metrics: parseMetricsFromUsage(read(responseBody, "usage")), + }); + } + } else { + child.log({ error: new Error("OpenAI Batch response body is missing") }); + } + } catch (error) { + child.log({ error }); + } finally { + child.end({ endTime }); + } +} + +async function readBatchInputs( + file: OpenAIBatchResultFile, + endpoint: string, +): Promise<{ + inputDigest: string; + inputs: Map; + issues: Error[]; +}> { + const inputs = new Map(); + const issues: Error[] = []; + const chunkDigests: string[] = []; + let digestChunk = ""; + try { + for await (const value of jsonlRecords(file, (issue) => + issues.push(issue), + )) { + const customId = read(value, "custom_id"); + const body = read(value, "body"); + if ( + !validCustomId(customId) || + inputs.has(customId) || + read(value, "method") !== "POST" || + read(value, "url") !== endpoint || + !isObject(body) + ) { + issues.push(new Error("OpenAI Batch input contains an invalid record")); + continue; + } + const canonicalRecord = JSON.stringify({ + body: canonicalizeJSON(body), + custom_id: customId, + }); + const framedRecord = `${canonicalRecord.length}:${canonicalRecord}`; + if ( + digestChunk.length > 0 && + digestChunk.length + framedRecord.length > INPUT_DIGEST_CHUNK_SIZE + ) { + chunkDigests.push( + digestHex( + await deterministicDigest("openai:batch:input:chunk", digestChunk), + 32, + ), + ); + digestChunk = ""; + } + digestChunk += framedRecord; + + let spanData: BatchInputRecord["spanData"]; + try { + spanData = extractOpenAIBatchInput(endpoint, body); + } catch (error) { + logBatchInstrumentationError("could not extract batch input", error); + } + inputs.set(customId, { customId, spanData }); + } + } catch (error) { + logBatchInstrumentationError("could not process input file", error); + issues.push(new Error("OpenAI Batch input file could not be processed")); + } + chunkDigests.push( + digestHex( + await deterministicDigest("openai:batch:input:chunk", digestChunk), + 32, + ), + ); + return { + inputDigest: digestHex( + await deterministicDigest("openai:batch:input", chunkDigests.join("")), + 32, + ), + inputs, + issues, + }; +} + +function canonicalizeJSON( + value: unknown, + ancestors: Set = new Set(), +): unknown { + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" + ) { + return value; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) { + throw new Error("OpenAI Batch input contains a non-finite number"); + } + return value; + } + if (Array.isArray(value)) { + if (ancestors.has(value)) { + throw new Error("OpenAI Batch input contains a circular value"); + } + ancestors.add(value); + try { + return value.map((entry) => canonicalizeJSON(entry, ancestors)); + } finally { + ancestors.delete(value); + } + } + if (!isObject(value)) { + throw new Error("OpenAI Batch input contains a non-JSON value"); + } + if (ancestors.has(value)) { + throw new Error("OpenAI Batch input contains a circular value"); + } + ancestors.add(value); + try { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => { + const entry = read(value, key); + if (entry === undefined) { + throw new Error("OpenAI Batch input contains an undefined value"); + } + return [key, canonicalizeJSON(entry, ancestors)]; + }), + ); + } finally { + ancestors.delete(value); + } +} + +async function completeResultFile({ + context, + endTime, + file, + inputs, + issues, + seen, + source, + taskParent, +}: { + context: BatchContext; + endTime: number; + file: OpenAIBatchResultFile; + inputs: Map; + issues: Error[]; + seen: Set; + source: BatchResultRecord["source"]; + taskParent: string; +}): Promise { + try { + for await (const value of jsonlRecords(file, (issue) => + issues.push(issue), + )) { + try { + const customId = read(value, "custom_id"); + if (!validCustomId(customId) || !isObject(value)) { + logBatchInstrumentationError( + "skipped result without custom_id", + value, + ); + issues.push( + new Error("OpenAI Batch result is missing a valid custom_id"), + ); + continue; + } + if (seen.has(customId)) { + logBatchInstrumentationError("skipped duplicate result", customId); + issues.push(new Error("OpenAI Batch result contains a duplicate")); + continue; + } + const input = inputs.get(customId); + if (!input) { + issues.push( + new Error("OpenAI Batch result does not match an input record"), + ); + continue; + } + seen.add(customId); + await completeBatchResult(context, taskParent, endTime, input, { + value, + source, + }); + } catch (error) { + logBatchInstrumentationError("skipped invalid result", error); + issues.push(new Error("OpenAI Batch result could not be processed")); + } + } + } catch (error) { + logBatchInstrumentationError(`could not process ${source} file`, error); + issues.push( + new Error(`OpenAI Batch ${source} file could not be processed`), + ); + } +} + +function terminalEndTime( + batch: OpenAIBatchLike, + webhookCreatedAt: number | undefined, + startTime: number, +): number { + const timestamp = + batch.status === "completed" + ? batch.completed_at + : batch.status === "failed" + ? batch.failed_at + : batch.status === "expired" + ? batch.expired_at + : batch.cancelled_at; + return typeof timestamp === "number" && + Number.isFinite(timestamp) && + timestamp >= startTime + ? timestamp + : typeof webhookCreatedAt === "number" && + Number.isFinite(webhookCreatedAt) && + webhookCreatedAt >= startTime + ? webhookCreatedAt + : Math.max(getCurrentUnixTimestamp(), startTime); +} + +async function completeBatch( + args: CompleteOpenAIBatchArgs, +): Promise { + const { batch, webhook } = args; + const batchId = read(batch, "id"); + const status = read(batch, "status"); + if ( + typeof batchId !== "string" || + typeof status !== "string" || + !TERMINAL_STATUSES.has(status) + ) { + return; + } + if ( + webhook && + (read(webhook, "type") !== `batch.${status}` || + read(read(webhook, "data"), "id") !== batchId) + ) { + logBatchInstrumentationError("ignored mismatched webhook", webhook); + return; + } + const context = await batchContextFromBatch(batch); + if (!context || !SUPPORTED_ENDPOINTS.has(context.endpoint)) { + return; + } + + const endTime = terminalEndTime( + batch, + webhook?.created_at, + context.startTime, + ); + const inputData = await readBatchInputs(args.input, context.endpoint); + const issues = [...inputData.issues]; + if ( + inputData.issues.length === 0 && + inputData.inputDigest !== context.inputDigest + ) { + issues.push( + new Error("OpenAI Batch input does not match the signed input file"), + ); + } + const requestTotal = read(read(batch, "request_counts"), "total"); + if ( + typeof requestTotal === "number" && + requestTotal !== inputData.inputs.size + ) { + issues.push( + new Error("OpenAI Batch input does not match its request count"), + ); + } + if (issues.length > 0) { + logBatchInstrumentationError("left batch spans pending", issues[0]); + return; + } + + const task = await startBatchSpan(context); + const taskParent = await task.export(); + const seen = new Set(); + await Promise.all([ + completeResultFile({ + context, + endTime, + file: args.outputFile, + inputs: inputData.inputs, + issues, + seen, + source: "output", + taskParent, + }), + completeResultFile({ + context, + endTime, + file: args.errorFile, + inputs: inputData.inputs, + issues, + seen, + source: "error", + taskParent, + }), + ]); + if (issues.length > 0) { + logBatchInstrumentationError("left batch spans pending", issues[0]); + return; + } + + const missing = [...inputData.inputs.values()].filter( + ({ customId }) => !seen.has(customId), + ); + if (status === "completed" && missing.length > 0) { + logBatchInstrumentationError( + "left batch spans pending", + new Error("OpenAI Batch result files are incomplete"), + ); + return; + } + for (const input of missing) { + try { + const child = await startBatchChild(context, taskParent, input); + child.log({ error: new Error(`OpenAI Batch ${status}`) }); + child.end({ endTime }); + } catch (error) { + logBatchInstrumentationError("could not complete missing result", error); + } + } + + if (status !== "completed") { + task.log({ error: new Error(`OpenAI Batch ${status}`) }); + } + task.end({ endTime }); +} + +export const interceptOpenAIBatchComplete: Parameters< + typeof openAIChannels.batchesComplete.intercept +>[0] = (target, thisArg, args) => + Promise.resolve(Reflect.apply(target, thisArg, args)).then(async (batch) => { + try { + await completeBatch({ ...args[0], batch }); + } catch (error) { + logBatchInstrumentationError("could not complete batch", error); + } + return batch; + }); diff --git a/js/src/instrumentation/plugins/openai-channels.ts b/js/src/instrumentation/plugins/openai-channels.ts index 4b1347609..1d27b0366 100644 --- a/js/src/instrumentation/plugins/openai-channels.ts +++ b/js/src/instrumentation/plugins/openai-channels.ts @@ -17,6 +17,11 @@ import type { OpenAIResponseCreateParams, OpenAIResponseStreamEvent, } from "../../vendor-sdk-types/openai"; +import type { + CompleteOpenAIBatchArgs, + CreateOpenAIBatchArgs, + OpenAIBatchLike, +} from "../../openai-batch-types"; type OpenAIChatSpanInfo = NonNullable["span_info"]>; @@ -31,6 +36,24 @@ type OpenAIResponsesChannelExtras = OpenAIChannelExtras; export const openAIChannels = defineChannels( "openai", { + batchesCreate: channel< + [CreateOpenAIBatchArgs], + OpenAIBatchLike, + OpenAIChannelExtras + >({ + channelName: "batches.create-resumable", + kind: "async", + }), + + batchesComplete: channel< + [CompleteOpenAIBatchArgs], + OpenAIBatchLike, + OpenAIChannelExtras + >({ + channelName: "batches.complete", + kind: "async", + }), + chatCompletionsCreate: channel< [OpenAIChatCreateParams], OpenAIChatCompletion | OpenAIChatStream, diff --git a/js/src/instrumentation/plugins/openai-plugin.test.ts b/js/src/instrumentation/plugins/openai-plugin.test.ts index 5f2b237fa..b55e56d8a 100644 --- a/js/src/instrumentation/plugins/openai-plugin.test.ts +++ b/js/src/instrumentation/plugins/openai-plugin.test.ts @@ -1,9 +1,9 @@ import { describe, it, expect } from "vitest"; import { parseMetricsFromUsage, - processImagesInOutput, aggregateChatCompletionChunks, } from "./openai-plugin"; +import { processImagesInOutput } from "./openai-span-data"; import { Attachment } from "../../logger"; describe("parseMetricsFromUsage", () => { diff --git a/js/src/instrumentation/plugins/openai-plugin.ts b/js/src/instrumentation/plugins/openai-plugin.ts index d1ca89dcc..0ee68a344 100644 --- a/js/src/instrumentation/plugins/openai-plugin.ts +++ b/js/src/instrumentation/plugins/openai-plugin.ts @@ -5,11 +5,19 @@ import { traceSyncStreamChannel, unsubscribeAll, } from "../core/channel-tracing"; -import { Attachment } from "../../logger"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; -import { processInputAttachments } from "../../wrappers/attachment-utils"; import { openAIChannels } from "./openai-channels"; +import { + extractOpenAIChatInput, + extractOpenAIResponsesInput, + extractOpenAIResponsesMetadata, + processImagesInOutput, +} from "./openai-span-data"; +import { + interceptOpenAIBatchComplete, + interceptOpenAIBatchCreate, +} from "./openai-batch-instrumentation"; import { BRAINTRUST_CACHED_STREAM_METRIC, getCachedMetricFromHeaders, @@ -38,18 +46,17 @@ export class OpenAIPlugin extends BasePlugin { } protected onEnable(): void { + this.unsubscribers.push( + openAIChannels.batchesCreate.intercept(interceptOpenAIBatchCreate), + openAIChannels.batchesComplete.intercept(interceptOpenAIBatchComplete), + ); + // Chat Completions - supports streaming this.unsubscribers.push( traceStreamingChannel(openAIChannels.chatCompletionsCreate, { name: "Chat Completion", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIChatInput(params), extractOutput: (result) => { return result?.choices; }, @@ -101,13 +108,7 @@ export class OpenAIPlugin extends BasePlugin { traceStreamingChannel(openAIChannels.betaChatCompletionsParse, { name: "Chat Completion", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIChatInput(params), extractOutput: (result) => { return result?.choices; }, @@ -131,13 +132,7 @@ export class OpenAIPlugin extends BasePlugin { traceSyncStreamChannel(openAIChannels.betaChatCompletionsStream, { name: "Chat Completion", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIChatInput(params), }), ); @@ -171,23 +166,11 @@ export class OpenAIPlugin extends BasePlugin { traceStreamingChannel(openAIChannels.responsesCreate, { name: "openai.responses.create", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractOutput: (result) => { return processImagesInOutput(result?.output); }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, + extractMetadata: (result) => extractOpenAIResponsesMetadata(result), extractMetrics: (result, startTime, endEvent) => { const metrics = withCachedMetric( parseMetricsFromUsage(result?.usage), @@ -208,13 +191,7 @@ export class OpenAIPlugin extends BasePlugin { traceSyncStreamChannel(openAIChannels.responsesStream, { name: "openai.responses.create", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractFromEvent: (event) => { if (event.type !== "response.completed" || !event.response) { return {}; @@ -243,23 +220,11 @@ export class OpenAIPlugin extends BasePlugin { traceStreamingChannel(openAIChannels.responsesParse, { name: "openai.responses.parse", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractOutput: (result) => { return processImagesInOutput(result?.output); }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, + extractMetadata: (result) => extractOpenAIResponsesMetadata(result), extractMetrics: (result, startTime, endEvent) => { const metrics = withCachedMetric( parseMetricsFromUsage(result?.usage), @@ -280,23 +245,11 @@ export class OpenAIPlugin extends BasePlugin { traceAsyncChannel(openAIChannels.responsesCompact, { name: "openai.responses.compact", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractOutput: (result) => { return processImagesInOutput(result?.output); }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, + extractMetadata: (result) => extractOpenAIResponsesMetadata(result), extractMetrics: (result, startTime, endEvent) => { const metrics = withCachedMetric( parseMetricsFromUsage(result?.usage), @@ -370,54 +323,6 @@ function withCachedMetric( }; } -/** - * Process output to convert base64 images to attachments. - * Used for Responses API image generation output. - */ -export function processImagesInOutput(output: any): any { - if (Array.isArray(output)) { - return output.map(processImagesInOutput); - } - - if (isObject(output)) { - if ( - output.type === "image_generation_call" && - output.result && - typeof output.result === "string" - ) { - const fileExtension = output.output_format || "png"; - const contentType = `image/${fileExtension}`; - - const baseFilename = - output.revised_prompt && typeof output.revised_prompt === "string" - ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") - : "generated_image"; - const filename = `${baseFilename}.${fileExtension}`; - - // Convert base64 string to Blob - const binaryString = atob(output.result); - const bytes = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - const blob = new Blob([bytes], { type: contentType }); - - const attachment = new Attachment({ - data: blob, - filename: filename, - contentType: contentType, - }); - - return { - ...output, - result: attachment, - }; - } - } - - return output; -} - function mergeLogprobTokens( existing: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], incoming: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], diff --git a/js/src/instrumentation/plugins/openai-span-data.ts b/js/src/instrumentation/plugins/openai-span-data.ts new file mode 100644 index 000000000..2562315ad --- /dev/null +++ b/js/src/instrumentation/plugins/openai-span-data.ts @@ -0,0 +1,120 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Attachment } from "../../logger"; +import { processInputAttachments } from "../../wrappers/attachment-utils"; +import { isObject } from "../../../util/index"; + +const OPENAI_METADATA_KEYS = [ + "model", + "temperature", + "top_p", + "max_tokens", + "frequency_penalty", + "presence_penalty", + "stop", + "response_format", + "tools", + "tool_choice", + "parallel_tool_calls", + "max_tool_calls", +] as const; + +function batchMetadata(params: Record) { + const metadata: Record = { provider: "openai" }; + for (const key of OPENAI_METADATA_KEYS) { + try { + const value = Reflect.get(params, key); + if (value !== undefined) { + metadata[key] = value; + } + } catch { + // Ignore hostile getters in provider input. + } + } + return metadata; +} + +export function extractOpenAIBatchInput( + endpoint: string, + params: Record, +): { input: unknown; metadata: Record } { + const input = + endpoint === "/v1/chat/completions" ? params.messages : params.input; + return { + input: processInputAttachments(input), + metadata: batchMetadata(params), + }; +} + +export function extractOpenAIChatInput(params: Record): { + input: unknown; + metadata: Record; +} { + const { messages, ...metadata } = params; + return { + input: processInputAttachments(messages), + metadata: { ...metadata, provider: "openai" }, + }; +} + +export function extractOpenAIResponsesInput(params: Record): { + input: unknown; + metadata: Record; +} { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; +} + +export function extractOpenAIResponsesMetadata( + result: Record | undefined, +): Record | undefined { + if (!result) { + return undefined; + } + const { output: _output, usage: _usage, ...metadata } = result; + return Object.keys(metadata).length > 0 ? metadata : undefined; +} + +/** Convert Responses API base64 image outputs to Braintrust attachments. */ +export function processImagesInOutput(output: any): any { + if (Array.isArray(output)) { + return output.map(processImagesInOutput); + } + + if ( + isObject(output) && + output.type === "image_generation_call" && + typeof output.result === "string" && + output.result + ) { + const fileExtension = output.output_format || "png"; + const contentType = `image/${fileExtension}`; + const baseFilename = + typeof output.revised_prompt === "string" + ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") + : "generated_image"; + let binaryString: string; + try { + binaryString = atob(output.result); + } catch { + return output; + } + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + return { + ...output, + result: new Attachment({ + data: new Blob([bytes], { type: contentType }), + filename: `${baseFilename}.${fileExtension}`, + contentType, + }), + }; + } + + return output; +} diff --git a/js/src/logger.ts b/js/src/logger.ts index 00d801a4a..8f83c053a 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -6328,6 +6328,22 @@ export function _internalStartSpanWithInitialMerge< InitialSpanWriteAsMergeArg).span; } +/** @internal Start a deterministic span under an exported object-only parent. */ +export function _internalStartSpanWithInitialMergeAndParentSpanIds< + IsAsyncFlush extends boolean = true, +>(args: StartSpanArgs & AsyncFlushArg & OptionalStateArg): Span { + return startSpanAndIsLogger( + { + ...args, + [INITIAL_SPAN_WRITE_AS_MERGE]: true, + } as StartSpanArgs & + AsyncFlushArg & + OptionalStateArg & + InitialSpanWriteAsMergeArg, + { useParentSpanIdsForObjectParent: true }, + ).span; +} + /** @internal Start a span with SDK-controlled context fields. */ export function _internalStartSpanWithContext< IsAsyncFlush extends boolean = true, @@ -6364,6 +6380,7 @@ function startSpanAndIsLogger( AsyncFlushArg & OptionalStateArg & InternalSpanContextArg, + internalOptions?: { useParentSpanIdsForObjectParent?: boolean }, ): { span: Span; isSyncFlushLogger: boolean } { const state = args?.state ?? _globalState; @@ -6381,7 +6398,7 @@ function startSpanAndIsLogger( parentObject instanceof SpanComponentsV3 || parentObject instanceof SpanComponentsV4 ) { - const parentSpanIds: ParentSpanIds | undefined = + const parentSpanIds: ParentSpanIds | MultiParentSpanIds | undefined = parentObject.data.row_id && parentSpanIdsUsable( parentObject.data.span_id, @@ -6391,7 +6408,9 @@ function startSpanAndIsLogger( spanId: parentObject.data.span_id, rootSpanId: parentObject.data.root_span_id, } - : undefined; + : internalOptions?.useParentSpanIdsForObjectParent + ? args?.parentSpanIds + : undefined; // The parent object/state are already resolved from `parent` above; drop // the raw `parent` so it isn't re-normalized. const { parent: _ignoredParent, ...spanArgs } = args ?? {}; diff --git a/js/src/openai-batch-types.ts b/js/src/openai-batch-types.ts new file mode 100644 index 000000000..f21a0540b --- /dev/null +++ b/js/src/openai-batch-types.ts @@ -0,0 +1,82 @@ +export type OpenAIBatchJSONL = + | string + | Iterable + | AsyncIterable; + +type OpenAIBatchResultFileContent = OpenAIBatchJSONL | Response; + +type OpenAIBatchResultFile = + | OpenAIBatchResultFileContent + | PromiseLike; + +type SupportedOpenAIBatchEndpoint = "/v1/chat/completions" | "/v1/responses"; + +export interface OpenAIBatchLike { + id: string; + endpoint: string; + input_file_id: string; + status: string; + completed_at?: number | null; + failed_at?: number | null; + expired_at?: number | null; + cancelled_at?: number | null; + metadata?: Record | null; + request_counts?: { + completed?: number; + failed?: number; + total?: number; + } | null; + [key: string]: unknown; +} + +export interface OpenAIBatchCreateParams { + input_file_id: string; + endpoint: SupportedOpenAIBatchEndpoint; + completion_window: "24h"; + metadata?: Record | null; + [key: string]: unknown; +} + +export interface OpenAIBatchCreateInputParams { + endpoint: SupportedOpenAIBatchEndpoint; + completion_window: "24h"; + metadata?: Record | null; + [key: string]: unknown; +} + +export interface OpenAIFileLike { + id: string; + [key: string]: unknown; +} + +export interface OpenAIBatchClientLike { + batches: { + create(params: OpenAIBatchCreateParams): PromiseLike; + }; +} + +export interface OpenAIBatchWebhookEvent { + type: + | "batch.completed" + | "batch.failed" + | "batch.expired" + | "batch.cancelled"; + created_at?: number; + data: { id: string }; + [key: string]: unknown; +} + +export interface CreateOpenAIBatchArgs { + client: OpenAIBatchClientLike; + inputFile: OpenAIFileLike; + input: OpenAIBatchJSONL; + params: OpenAIBatchCreateInputParams; +} + +export interface CompleteOpenAIBatchArgs { + batch: TBatch; + input: OpenAIBatchResultFile; + outputFile?: OpenAIBatchResultFile; + errorFile?: OpenAIBatchResultFile; + webhook?: OpenAIBatchWebhookEvent; +} diff --git a/js/src/openai-batch.ts b/js/src/openai-batch.ts new file mode 100644 index 000000000..510997816 --- /dev/null +++ b/js/src/openai-batch.ts @@ -0,0 +1,65 @@ +import { openAIChannels } from "./instrumentation/plugins/openai-channels"; +import type { + CompleteOpenAIBatchArgs, + CreateOpenAIBatchArgs, + OpenAIBatchLike, +} from "./openai-batch-types"; + +/** + * Create an OpenAI Batch and start pending Braintrust spans for its requests. + * + * Supported endpoints are `/v1/chat/completions` and `/v1/responses`. + */ +export async function createOpenAIBatch( + args: CreateOpenAIBatchArgs, +): Promise { + const batch = await openAIChannels.batchesCreate.invoke( + async (input) => + await input.client.batches.create({ + ...input.params, + input_file_id: input.inputFile.id, + }), + undefined, + [args], + {}, + ); + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return batch as TBatch; +} + +/** + * Complete pending OpenAI Batch spans from caller-supplied Batch API data. + * + * This helper performs no OpenAI API requests. Pass the Responses or promises + * returned by `client.files.content()` for the input and result files, JSONL + * strings, or iterables of parsed records. + */ +export async function completeOpenAIBatch( + args: CompleteOpenAIBatchArgs, +): Promise { + const input = Promise.resolve(args.input); + const outputFile = + args.outputFile === undefined + ? undefined + : Promise.resolve(args.outputFile); + const errorFile = + args.errorFile === undefined ? undefined : Promise.resolve(args.errorFile); + + // Content requests are commonly started before this helper is called. Attach + // rejection handlers immediately so early no-op paths and disabled + // instrumentation cannot leave caller-supplied promises unhandled. + void input.catch(() => undefined); + void outputFile?.catch(() => undefined); + void errorFile?.catch(() => undefined); + + const batch = await openAIChannels.batchesComplete.invoke( + async (input) => input.batch, + undefined, + [{ ...args, input, outputFile, errorFile }], + {}, + ); + // The completion channel preserves the exact batch object supplied by the + // caller, including fields from a newer OpenAI SDK version. + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return batch as TBatch; +} diff --git a/js/src/openai-promise-utils.test.ts b/js/src/openai-promise-utils.test.ts new file mode 100644 index 000000000..dc474c316 --- /dev/null +++ b/js/src/openai-promise-utils.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from "vitest"; +import { openAIChannels } from "./instrumentation/plugins/openai-channels"; +import { wrapOpenAI } from "./wrappers/oai"; +import { + type APIPromise, + createLazyAPIPromise, +} from "./wrappers/openai-promise-utils"; + +function apiPromise(data: T, requestId: string): APIPromise { + const response = new Response(JSON.stringify(data), { + headers: { "content-type": "application/json" }, + }); + const promise = Promise.resolve(data) as APIPromise; + promise.withResponse = async () => ({ + data, + response, + request_id: requestId, + }); + promise.asResponse = async () => response; + return promise; +} + +describe("OpenAI API promise wrappers", () => { + it("forwards asResponse without parsing the response body", async () => { + const response = new Response(JSON.stringify({ value: "raw" }), { + headers: { "content-type": "application/json" }, + }); + const nativePromise = apiPromise({ value: "parsed" }, "req_raw"); + let tracedWithResponseCalls = 0; + let tracedAsResponseCalls = 0; + const wrapped = createLazyAPIPromise( + async () => { + tracedWithResponseCalls++; + return { data: { value: "parsed" }, response }; + }, + async () => { + tracedAsResponseCalls++; + return response; + }, + () => nativePromise, + ); + + const raw = await wrapped.asResponse(); + expect(tracedWithResponseCalls).toBe(0); + expect(tracedAsResponseCalls).toBe(1); + await expect(raw.json()).resolves.toEqual({ value: "raw" }); + }); + + it("does not trace the request again when parsed after asResponse", async () => { + const response = new Response(JSON.stringify({ value: "raw" }), { + headers: { "content-type": "application/json" }, + }); + const nativePromise = apiPromise({ value: "parsed" }, "req_raw"); + let tracedWithResponseCalls = 0; + let tracedAsResponseCalls = 0; + const wrapped = createLazyAPIPromise( + async () => { + tracedWithResponseCalls++; + return { data: { value: "parsed" }, response }; + }, + async () => { + tracedAsResponseCalls++; + return response; + }, + () => nativePromise, + ); + + await wrapped.asResponse(); + await expect(wrapped).resolves.toEqual({ value: "parsed" }); + expect(tracedWithResponseCalls).toBe(0); + expect(tracedAsResponseCalls).toBe(1); + }); + + it("dispatches the OpenAI channel lifecycle for asResponse-only calls", async () => { + const phases: string[] = []; + let result: unknown; + const tracingChannel = + openAIChannels.chatCompletionsCreate.tracingChannel(); + const handlers = { + asyncEnd: (event: { result?: unknown }) => { + phases.push("asyncEnd"); + result = event.result; + }, + error: () => phases.push("error"), + start: () => phases.push("start"), + }; + tracingChannel.subscribe(handlers); + + try { + const client = wrapOpenAI({ + chat: { + completions: { + create: (_params: { messages: unknown }) => + apiPromise({ choices: [] }, "req_raw"), + }, + }, + embeddings: { create: () => apiPromise({ data: [] }, "req_embed") }, + moderations: { + create: () => apiPromise({ results: [] }, "req_moderation"), + }, + }); + + const response = await client.chat.completions + .create({ messages: [] }) + .asResponse(); + expect(response.bodyUsed).toBe(false); + await new Promise((resolve) => setTimeout(resolve, 0)); + } finally { + tracingChannel.unsubscribe(handlers); + } + + expect(phases).toEqual(["start", "asyncEnd"]); + expect(result).toBeUndefined(); + }); + + it("does not consume a raw response body and preserves cancellation", async () => { + let cancelled = false; + let pulls = 0; + const response = new Response( + new ReadableStream( + { + pull(controller) { + pulls++; + controller.enqueue( + new TextEncoder().encode(JSON.stringify({ choices: [] })), + ); + }, + cancel() { + cancelled = true; + }, + }, + { highWaterMark: 0 }, + ), + { headers: { "content-type": "application/json" } }, + ); + const promise: APIPromise<{ choices: unknown[] }> = Object.assign( + Promise.resolve({ choices: [] as unknown[] }), + { + asResponse: async () => response, + withResponse: async () => ({ data: { choices: [] }, response }), + }, + ); + const phases: string[] = []; + const tracingChannel = + openAIChannels.chatCompletionsCreate.tracingChannel(); + const handlers = { + asyncEnd: () => phases.push("asyncEnd"), + error: () => phases.push("error"), + start: () => phases.push("start"), + }; + tracingChannel.subscribe(handlers); + + try { + const client = wrapOpenAI({ + chat: { + completions: { + create: (_params: { messages: unknown }) => promise, + }, + }, + embeddings: { create: () => apiPromise({ data: [] }, "req_embed") }, + moderations: { + create: () => apiPromise({ results: [] }, "req_moderation"), + }, + }); + + const raw = await client.chat.completions + .create({ messages: [] }) + .asResponse(); + expect(raw).toBe(response); + expect(raw.bodyUsed).toBe(false); + expect(pulls).toBe(0); + expect(phases).toEqual(["start", "asyncEnd"]); + + const reader = raw.body?.getReader(); + await reader?.read(); + await reader?.cancel(); + expect(cancelled).toBe(true); + } finally { + tracingChannel.unsubscribe(handlers); + } + }); + + it.each([false, true])( + "preserves request_id for chat responses with stream=%s", + async (stream) => { + const client = wrapOpenAI({ + chat: { + completions: { + create: (_params: { messages: unknown; stream?: boolean }) => + apiPromise({ choices: [] }, "req_chat"), + }, + }, + embeddings: { create: () => apiPromise({ data: [] }, "req_embed") }, + moderations: { + create: () => apiPromise({ results: [] }, "req_moderation"), + }, + }); + + await expect( + client.chat.completions.create({ messages: [], stream }).withResponse(), + ).resolves.toMatchObject({ request_id: "req_chat" }); + }, + ); +}); diff --git a/js/src/vendor-sdk-types/openai-common.ts b/js/src/vendor-sdk-types/openai-common.ts index d2dca2323..15f88a3c9 100644 --- a/js/src/vendor-sdk-types/openai-common.ts +++ b/js/src/vendor-sdk-types/openai-common.ts @@ -8,11 +8,13 @@ export interface OpenAIAPIPromise extends Promise { withResponse(): Promise>; + asResponse(): Promise; } export interface OpenAIWithResponse { data: T; response: Response; + request_id?: string | null; } // Requests diff --git a/js/src/wrappers/oai.ts b/js/src/wrappers/oai.ts index b2fa0850e..1478b0569 100644 --- a/js/src/wrappers/oai.ts +++ b/js/src/wrappers/oai.ts @@ -27,6 +27,7 @@ import { createLazyAPIPromise, EnhancedResponse, splitSpanInfo, + tracePromiseAsResponse, tracePromiseWithResponse, } from "./openai-promise-utils"; import { OpenAIV4Client } from "../vendor-sdk-types/openai-v4"; @@ -113,7 +114,6 @@ export function wrapOpenAIv4(openai: T): T { OpenAIModerationCreateParams, OpenAIModerationResponse >(typedOpenai.moderations, wrapModerations); - let betaProxy: OpenAIClient["beta"]; if (typedOpenai.beta?.chat?.completions?.stream) { const betaChatCompletionProxy = new Proxy( @@ -148,7 +148,6 @@ export function wrapOpenAIv4(openai: T): T { }); } - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const topLevelProxy = new Proxy(typedOpenai, { get(target, name) { switch (name) { @@ -254,6 +253,12 @@ function wrapChatCompletion< // to avoid unhandled rejections when the underlying OpenAI call fails immediately. // Without lazy execution, the promise chain starts before error handlers are attached. let executionPromise: Promise> | null = null; + let apiPromise: APIPromise | null = null; + + const getAPIPromise = (): APIPromise => { + apiPromise ??= completion(params, options); + return apiPromise; + }; const ensureExecuted = (): Promise> => { if (!executionPromise) { @@ -265,37 +270,48 @@ function wrapChatCompletion< ); if (params.stream) { - const completionPromise = completion( - params, - options, - ) as APIPromise; - const { data, response } = await tracePromiseWithResponse( - openAIChannels.chatCompletionsCreate, - traceContext, - completionPromise, - ); + const completionPromise = + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + getAPIPromise() as APIPromise; + const { data, response, request_id } = + await tracePromiseWithResponse( + openAIChannels.chatCompletionsCreate, + traceContext, + completionPromise, + ); // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return { data: data as C, response }; + return { data: data as C, response, request_id }; } - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const completionResponse = completion( - params, - options, - ) as APIPromise; - const { data, response } = await tracePromiseWithResponse( + const completionResponse = + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + getAPIPromise() as APIPromise; + const { data, response, request_id } = await tracePromiseWithResponse( openAIChannels.chatCompletionsCreate, traceContext, completionResponse, ); // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return { data: data as C, response }; + return { data: data as C, response, request_id }; })(); } return executionPromise; }; - return createLazyAPIPromise(ensureExecuted); + return createLazyAPIPromise( + ensureExecuted, + () => + tracePromiseAsResponse( + openAIChannels.chatCompletionsCreate, + createChannelContext( + openAIChannels.chatCompletionsCreate, + params, + span_info, + ), + getAPIPromise(), + ), + getAPIPromise, + ); }; } @@ -338,6 +354,11 @@ function wrapApiCreateWithChannel< >(allParams); let executionPromise: Promise>> | null = null; + let apiPromise: APIPromise> | null = null; + const getAPIPromise = () => { + apiPromise ??= create(params, options); + return apiPromise; + }; const ensureExecuted = () => { if (!executionPromise) { executionPromise = (async () => { @@ -345,13 +366,22 @@ function wrapApiCreateWithChannel< return tracePromiseWithResponse( channel, traceContext, - create(params, options), + getAPIPromise(), ); })(); } return executionPromise; }; - return createLazyAPIPromise(ensureExecuted); + return createLazyAPIPromise( + ensureExecuted, + () => + tracePromiseAsResponse( + channel, + createChannelContext(channel, params, span_info), + getAPIPromise(), + ), + getAPIPromise, + ); }; } diff --git a/js/src/wrappers/oai_responses.ts b/js/src/wrappers/oai_responses.ts index 8e936ebbc..466856286 100644 --- a/js/src/wrappers/oai_responses.ts +++ b/js/src/wrappers/oai_responses.ts @@ -11,6 +11,7 @@ import { createLazyAPIPromise, EnhancedResponse, splitSpanInfo, + tracePromiseAsResponse, tracePromiseWithResponse, } from "./openai-promise-utils"; @@ -78,6 +79,12 @@ function wrapResponsesAsync< let executionPromise: Promise>> | null = null; + let apiPromise: APIPromise> | null = null; + + const getAPIPromise = () => { + apiPromise ??= target(params, options); + return apiPromise; + }; const ensureExecuted = (): Promise< EnhancedResponse> @@ -85,15 +92,27 @@ function wrapResponsesAsync< if (!executionPromise) { executionPromise = (async () => { const traceContext = createChannelContext(channel, params, span_info); - const apiPromise = target(params, options); - return tracePromiseWithResponse(channel, traceContext, apiPromise); + return tracePromiseWithResponse( + channel, + traceContext, + getAPIPromise(), + ); })(); } return executionPromise; }; - return createLazyAPIPromise(ensureExecuted); + return createLazyAPIPromise( + ensureExecuted, + () => + tracePromiseAsResponse( + channel, + createChannelContext(channel, params, span_info), + getAPIPromise(), + ), + getAPIPromise, + ); }; } diff --git a/js/src/wrappers/openai-promise-utils.ts b/js/src/wrappers/openai-promise-utils.ts index afa23586c..9ba00d0bd 100644 --- a/js/src/wrappers/openai-promise-utils.ts +++ b/js/src/wrappers/openai-promise-utils.ts @@ -11,10 +11,12 @@ import type { export type EnhancedResponse = { response: Response; data: T; + request_id?: string | null; }; export interface APIPromise extends Promise { withResponse(): Promise>; + asResponse(): Promise; } type ChannelContext = @@ -71,19 +73,78 @@ export async function tracePromiseWithResponse< throw new Error("Expected withResponse() to provide response"); } - return { data, response: enhancedResponse.response }; + return { + data, + response: enhancedResponse.response, + request_id: enhancedResponse.request_id, + }; +} + +export async function tracePromiseAsResponse< + TChannel extends OpenAIAsyncChannel, + TResult extends ResultOf, +>( + channel: TChannel, + traceContext: ChannelContext, + apiPromise: APIPromise, +): Promise { + const tracePromise = + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + channel.tracePromise as unknown as ( + fn: () => Promise, + context: ChannelContext, + ) => Promise; + + let response: Response | undefined; + await tracePromise(async () => { + response = await apiPromise.asResponse(); + traceContext.response = response; + return undefined; + }, traceContext); + + if (!response) { + throw new Error("Expected asResponse() to provide response"); + } + return response; } export function createLazyAPIPromise( ensureExecuted: () => Promise>, + ensureResponse: () => Promise, + getAPIPromise: () => APIPromise, ): APIPromise { - let dataPromise: Promise | null = null; + let firstConsumption: "data" | "response" | undefined; + let enhancedResponsePromise: Promise> | undefined; + let dataPromise: Promise | undefined; + let responsePromise: Promise | undefined; + + const withResponse = () => { + firstConsumption ??= "data"; + enhancedResponsePromise ??= + firstConsumption === "data" + ? ensureExecuted() + : getAPIPromise().withResponse(); + return enhancedResponsePromise; + }; + + const asResponse = () => { + firstConsumption ??= "response"; + responsePromise ??= + firstConsumption === "response" + ? ensureResponse() + : getAPIPromise().asResponse(); + return responsePromise; + }; // eslint-disable-next-line @typescript-eslint/consistent-type-assertions return new Proxy({} as APIPromise, { get(target, prop, receiver) { if (prop === "withResponse") { - return () => ensureExecuted(); + return withResponse; + } + + if (prop === "asResponse") { + return asResponse; } if ( @@ -92,9 +153,7 @@ export function createLazyAPIPromise( prop === "finally" || prop in Promise.prototype ) { - if (!dataPromise) { - dataPromise = ensureExecuted().then((result) => result.data); - } + dataPromise ??= withResponse().then((result) => result.data); const value = Reflect.get(dataPromise, prop, receiver); return typeof value === "function" ? value.bind(dataPromise) : value; } From bdcae71df2df276b1ae9f847867e98b24b780226 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:59:07 +0000 Subject: [PATCH 2/6] feat: Add OpenAI Batch API Instrumentation --- AGENTS.md | 4 + .../openai-v4-auto-hook.span-tree.json | 1355 ++++++++++++--- .../openai-v4-auto-hook.span-tree.txt | 1334 ++++++++++++--- .../openai-v4-latest-auto-hook.span-tree.json | 1355 ++++++++++++--- .../openai-v4-latest-auto-hook.span-tree.txt | 1334 ++++++++++++--- .../openai-v4-latest-wrapped.span-tree.json | 1355 ++++++++++++--- .../openai-v4-latest-wrapped.span-tree.txt | 1334 ++++++++++++--- .../openai-v4-wrapped.span-tree.json | 1355 ++++++++++++--- .../openai-v4-wrapped.span-tree.txt | 1334 ++++++++++++--- .../openai-v5-auto-hook.span-tree.json | 1355 ++++++++++++--- .../openai-v5-auto-hook.span-tree.txt | 1334 ++++++++++++--- .../openai-v5-latest-auto-hook.span-tree.json | 1355 ++++++++++++--- .../openai-v5-latest-auto-hook.span-tree.txt | 1334 ++++++++++++--- .../openai-v5-latest-wrapped.span-tree.json | 1355 ++++++++++++--- .../openai-v5-latest-wrapped.span-tree.txt | 1334 ++++++++++++--- .../openai-v5-wrapped.span-tree.json | 1355 ++++++++++++--- .../openai-v5-wrapped.span-tree.txt | 1334 ++++++++++++--- .../openai-v6-auto-hook.span-tree.json | 1524 ++++++++++++++--- .../openai-v6-auto-hook.span-tree.txt | 1503 +++++++++++++--- .../openai-v6-latest-auto-hook.span-tree.json | 1524 ++++++++++++++--- .../openai-v6-latest-auto-hook.span-tree.txt | 1503 +++++++++++++--- .../openai-v6-latest-wrapped.span-tree.json | 1524 ++++++++++++++--- .../openai-v6-latest-wrapped.span-tree.txt | 1503 +++++++++++++--- .../openai-v6-wrapped.span-tree.json | 1524 ++++++++++++++--- .../openai-v6-wrapped.span-tree.txt | 1503 +++++++++++++--- .../openai-instrumentation/assertions.ts | 367 ++-- .../openai-instrumentation/scenario.impl.mjs | 146 ++ js/src/exports.ts | 2 + js/src/instrumentation/braintrust-plugin.ts | 4 +- .../plugins/openai-batch-constants.ts | 1 + .../openai-batch-instrumentation.test.ts | 722 ++++++++ .../plugins/openai-batch-instrumentation.ts | 1120 ++++++++++++ .../plugins/openai-channels.ts | 23 + .../plugins/openai-plugin.test.ts | 2 +- .../instrumentation/plugins/openai-plugin.ts | 145 +- .../plugins/openai-span-data.ts | 120 ++ js/src/logger.ts | 23 +- js/src/openai-batch-types.ts | 82 + js/src/openai-batch.ts | 65 + js/src/openai-promise-utils.test.ts | 204 +++ js/src/vendor-sdk-types/openai-common.ts | 2 + js/src/wrappers/oai.ts | 74 +- js/src/wrappers/oai_responses.ts | 25 +- js/src/wrappers/openai-promise-utils.ts | 71 +- 44 files changed, 30373 insertions(+), 6449 deletions(-) create mode 100644 js/src/instrumentation/plugins/openai-batch-constants.ts create mode 100644 js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts create mode 100644 js/src/instrumentation/plugins/openai-batch-instrumentation.ts create mode 100644 js/src/instrumentation/plugins/openai-span-data.ts create mode 100644 js/src/openai-batch-types.ts create mode 100644 js/src/openai-batch.ts create mode 100644 js/src/openai-promise-utils.test.ts diff --git a/AGENTS.md b/AGENTS.md index 6338e8830..91b5f76f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,10 @@ mise install # Install toolchain and dependencies pnpm run build # Build all workspace packages (from repo root) ``` +## Public API + +Keep public exports minimal. Generally, export only the requested runtime APIs and do not export types unless explicitly requested. + ## Instrumentation Use the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch. diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json index 49e7bb1d8..7d6ea7f6e 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +892,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +1002,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1112,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1325,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1431,138 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 27, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 73 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt index cbce77cc0..dcb056028 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt @@ -1,312 +1,586 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +589,816 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 27, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 73 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json index 49e7bb1d8..ba468e90c 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +892,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +1002,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1112,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1325,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1431,138 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 19, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 65 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt index cbce77cc0..de6f2dab6 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt @@ -1,312 +1,586 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +589,816 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 19, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 65 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json index 49e7bb1d8..920f620f5 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +884,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +994,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1104,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1317,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1423,146 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 19, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 65 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt index cbce77cc0..c13222807 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt @@ -1,312 +1,578 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +581,824 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 19, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 65 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json index 49e7bb1d8..d7968e46f 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +884,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +994,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1104,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1317,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1423,146 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 27, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 73 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt index cbce77cc0..5c19f6e7b 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt @@ -1,312 +1,578 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +581,824 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 27, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 73 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json index 49e7bb1d8..b5726d1d3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +892,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +1002,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1112,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1325,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1431,138 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 21, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 67 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt index cbce77cc0..93c5f56e6 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt @@ -1,312 +1,586 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +589,816 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 21, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 67 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json index 49e7bb1d8..e71a94318 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +586,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +642,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +651,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +673,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +892,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +1002,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1112,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1219,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1325,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1431,138 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 45, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 91 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.23.2", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt index cbce77cc0..917e99cc2 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt @@ -1,312 +1,586 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.23.2", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +589,816 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 45, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 91 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json index 49e7bb1d8..82c330f57 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +884,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +994,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1104,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1317,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1423,146 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 45, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 91 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.23.2", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt index cbce77cc0..0d5d65a88 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt @@ -1,312 +1,578 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.23.2", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +581,824 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 45, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 91 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json index 49e7bb1d8..a7acdaec3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +218,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +276,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +334,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +428,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,36 +515,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -327,34 +589,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,9 +634,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -375,19 +643,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -397,31 +665,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -431,38 +884,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -472,38 +994,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -513,38 +1104,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -554,38 +1211,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -595,38 +1317,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -636,54 +1423,146 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 together gives us 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 together gives us 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 21, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 67 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt index cbce77cc0..9755e3a63 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt @@ -1,312 +1,578 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -315,232 +581,824 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } └── openai-responses-parse-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.parse [llm] - input: { - "kind": "text" - } + input: "What is 20 + 4?" output: [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 together gives us 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ] metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 together gives us 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 21, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 67 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json index a9eea3198..5b5261c56 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +731,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +787,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +796,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +818,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +1037,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1147,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1257,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1364,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1470,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1576,126 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 30, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 76 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1707,82 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 220, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 353 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt index cea5718a8..b5864f6a1 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt @@ -1,374 +1,715 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-image-attachment" + │ "operation": "chat-image-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Describe this image in three words or fewer.", + │ "type": "text" + │ }, + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ }, + │ "type": "image_url" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Solid red.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 8516, + │ "time_to_first_token": 0, + │ "tokens": 8519 │ } ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "operation": "chat-pdf-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Summarize this document in one short phrase.", + │ "type": "text" + │ }, + │ { + │ "file": { + │ "file_data": { + │ "content_type": "application/pdf", + │ "filename": "document.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "filename": "document.pdf" + │ }, + │ "type": "file" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "length", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 24, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 236, + │ "time_to_first_token": 0, + │ "tokens": 260 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -377,279 +718,883 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" + │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-parse" + │ "operation": "responses-parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 20 + 4?" │ output: [ │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "type": "output_text" + │ } │ ], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": null, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "description": null, + │ "name": "NumberAnswer", + │ "schema": { + │ "additionalProperties": false, + │ "properties": { + │ "reasoning": { + │ "type": "string" + │ }, + │ "value": { + │ "type": "integer" + │ } + │ }, + │ "required": [ + │ "value", + │ "reasoning" + │ ], + │ "type": "object" + │ }, + │ "strict": true, + │ "type": "json_schema" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 30, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 46, + │ "time_to_first_token": 0, + │ "tokens": 76 │ } └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.compact [llm] input: [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ] output: [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "id": "", "type": "compaction" } ] metadata: { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 220, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 353 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json index a9eea3198..69ed5df8d 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,66 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +731,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +787,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +796,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +818,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +1037,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1147,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1257,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1364,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1470,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1576,126 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 17, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 63 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1707,82 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 190, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 323 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.48.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt index cea5718a8..9e8dca063 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt @@ -1,374 +1,715 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.48.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-image-attachment" + │ "operation": "chat-image-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Describe this image in three words or fewer.", + │ "type": "text" + │ }, + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ }, + │ "type": "image_url" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Solid red.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 8516, + │ "time_to_first_token": 0, + │ "tokens": 8519 │ } ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "operation": "chat-pdf-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Summarize this document in one short phrase.", + │ "type": "text" + │ }, + │ { + │ "file": { + │ "file_data": { + │ "content_type": "application/pdf", + │ "filename": "document.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "filename": "document.pdf" + │ }, + │ "type": "file" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "length", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 24, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 236, + │ "time_to_first_token": 0, + │ "tokens": 260 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -377,279 +718,883 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" + │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-parse" + │ "operation": "responses-parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 20 + 4?" │ output: [ │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "type": "output_text" + │ } │ ], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": null, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "description": null, + │ "name": "NumberAnswer", + │ "schema": { + │ "additionalProperties": false, + │ "properties": { + │ "reasoning": { + │ "type": "string" + │ }, + │ "value": { + │ "type": "integer" + │ } + │ }, + │ "required": [ + │ "value", + │ "reasoning" + │ ], + │ "type": "object" + │ }, + │ "strict": true, + │ "type": "json_schema" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 17, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 46, + │ "time_to_first_token": 0, + │ "tokens": 63 │ } └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.compact [llm] input: [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ] output: [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "id": "", "type": "compaction" } ] metadata: { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 190, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 323 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json index a9eea3198..43b2d172d 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +734,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +779,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +788,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +810,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +1029,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1139,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1249,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1356,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1462,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1568,134 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "20 plus 4 equals 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "20 plus 4 equals 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 17, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 63 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1707,82 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 190, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 323 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.48.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt index cea5718a8..4deb2bed8 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt @@ -1,374 +1,707 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.48.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-image-attachment" + │ "operation": "chat-image-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Describe this image in three words or fewer.", + │ "type": "text" + │ }, + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ }, + │ "type": "image_url" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Solid red.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 8516, + │ "time_to_first_token": 0, + │ "tokens": 8519 │ } ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "operation": "chat-pdf-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Summarize this document in one short phrase.", + │ "type": "text" + │ }, + │ { + │ "file": { + │ "file_data": { + │ "content_type": "application/pdf", + │ "filename": "document.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "filename": "document.pdf" + │ }, + │ "type": "file" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "length", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 24, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 236, + │ "time_to_first_token": 0, + │ "tokens": 260 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -377,279 +710,891 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" + │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 │ } ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-parse" + │ "operation": "responses-parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 20 + 4?" │ output: [ │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "parsed": { + │ "reasoning": "20 plus 4 equals 24.", + │ "value": 24 + │ }, + │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "type": "output_text" + │ } │ ], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": null, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_parsed": { + │ "reasoning": "20 plus 4 equals 24.", + │ "value": 24 + │ }, + │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "description": null, + │ "name": "NumberAnswer", + │ "schema": { + │ "additionalProperties": false, + │ "properties": { + │ "reasoning": { + │ "type": "string" + │ }, + │ "value": { + │ "type": "integer" + │ } + │ }, + │ "required": [ + │ "value", + │ "reasoning" + │ ], + │ "type": "object" + │ }, + │ "strict": true, + │ "type": "json_schema" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 17, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 46, + │ "time_to_first_token": 0, + │ "tokens": 63 │ } └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.compact [llm] input: [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ] output: [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "id": "", "type": "compaction" } ] metadata: { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 190, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 323 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json index a9eea3198..034953fc9 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json @@ -13,34 +13,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly OK.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat" + "operation": "chat", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -52,34 +69,51 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly FOUR.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-with-response" + "operation": "chat-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -91,34 +125,67 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Describe this image in three words or fewer.", + "type": "text" + }, + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + }, + "type": "image_url" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Solid red.", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 8516, + "time_to_first_token": 0, + "tokens": 8519 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-image-attachment" + "operation": "chat-image-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -130,34 +197,68 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "Summarize this document in one short phrase.", + "type": "text" + }, + { + "file": { + "file_data": { + "content_type": "application/pdf", + "filename": "document.pdf", + "key": "", + "type": "braintrust_attachment" + }, + "filename": "document.pdf" + }, + "type": "file" + } + ], "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "length", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 24, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 236, + "time_to_first_token": 0, + "tokens": 260 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-pdf-attachment" + "operation": "chat-pdf-attachment", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -169,34 +270,88 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "chat-tool" + "operation": "chat-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -208,34 +363,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream" + "operation": "stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -247,34 +421,53 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly STREAM-WITH-RESPONSE.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-with-response" + "operation": "stream-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -286,34 +479,89 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } } ], "metadata": { + "max_tokens": 64, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "The location to get weather for", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 7, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 77, + "time_to_first_token": 0, + "tokens": 84 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-tool" + "operation": "stream-tool", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -325,34 +573,82 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with a refusal stream fixture.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } } ], "metadata": { + "logprobs": true, + "max_tokens": 12, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "stream-fixture" + "operation": "stream-fixture", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -364,36 +660,69 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "What is 2 + 2?", "role": "user" } ], "output": [ { - "json_keys": [ - "answer" - ], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } } ], "metadata": { "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } }, "metrics": { - "has_time_to_first_token": true + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "parse" + "operation": "parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -405,34 +734,42 @@ "children": [], "input": [ { - "content_kind": "string", + "content": "Reply with exactly SYNC STREAM.", "role": "user" } ], "output": [ { - "json_keys": [], - "role": "assistant" + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } } ], "metadata": { + "max_tokens": 24, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "provider": "openai", + "temperature": 0 }, "metrics": { - "has_time_to_first_token": true + "time_to_first_token": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "sync-stream" + "operation": "sync-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -442,9 +779,7 @@ "name": "Embedding", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Paris", "output": { "embedding_length": 1536 }, @@ -453,19 +788,19 @@ "provider": "openai" }, "metrics": { - "has_time_to_first_token": false + "prompt_tokens": 1, + "tokens": 1 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "embeddings" + "operation": "embeddings", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -475,31 +810,216 @@ "name": "Moderation", "type": "llm", "children": [], - "input": { - "kind": "text" - }, - "output": { - "flagged_count": 0, - "result_count": 1 - }, + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], "metadata": { "model": "omni-moderation-2024-09-26", "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false } } ], - "input": { - "kind": "undefined" + "metadata": { + "operation": "moderations", + "testRunId": "" }, - "output": null, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 + } + }, + { + "name": "openai-batch-operation", + "children": [ + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], "metadata": { - "operation": "moderations" + "operation": "batch", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -509,38 +1029,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARIS.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses" + "operation": "responses", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -550,38 +1139,107 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 2 + 2? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-with-response" + "operation": "responses-with-response", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -591,38 +1249,104 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly RESPONSE STREAM.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-create-stream" + "operation": "responses-create-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -632,38 +1356,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 6 x 6? Reply with just the number.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream" + "operation": "responses-stream", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -673,38 +1462,103 @@ "name": "openai.responses.create", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "Reply with exactly PARTIAL.", "output": [ { - "content_types": [ - "output_text" + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } ], - "json_keys": [], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-stream-partial" + "operation": "responses-stream-partial", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -714,41 +1568,134 @@ "name": "openai.responses.parse", "type": "llm", "children": [], - "input": { - "kind": "text" - }, + "input": "What is 20 + 4?", "output": [ { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "type": "output_text" + } ], + "id": "", "role": "assistant", "status": "completed", "type": "message" } ], "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 30, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 76 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-parse" + "operation": "responses-parse", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } }, { @@ -760,63 +1707,82 @@ "children": [], "input": [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ], "output": [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "id": "", "type": "compaction" } ], "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" }, "metrics": { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 220, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 353 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } } ], - "input": { - "kind": "undefined" - }, - "output": null, "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" }, - "metrics": { - "has_time_to_first_token": false + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } } ] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt index cea5718a8..6475d02dc 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt @@ -1,374 +1,707 @@ span_tree: └── openai-instrumentation-root [task] - input: { - "kind": "undefined" - } - output: null metadata: { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 } ├── openai-chat-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat" + │ "operation": "chat", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly OK.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-with-response" + │ "operation": "chat-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly FOUR.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 │ } ├── openai-chat-image-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-image-attachment" + │ "operation": "chat-image-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Describe this image in three words or fewer.", + │ "type": "text" + │ }, + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ }, + │ "type": "image_url" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Solid red.", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 8516, + │ "time_to_first_token": 0, + │ "tokens": 8519 │ } ├── openai-chat-pdf-attachment-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-pdf-attachment" + │ "operation": "chat-pdf-attachment", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "blocks", + │ "content": [ + │ { + │ "text": "Summarize this document in one short phrase.", + │ "type": "text" + │ }, + │ { + │ "file": { + │ "file_data": { + │ "content_type": "application/pdf", + │ "filename": "document.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "filename": "document.pdf" + │ }, + │ "type": "file" + │ } + │ ], │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "length", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 24, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 236, + │ "time_to_first_token": 0, + │ "tokens": 260 │ } ├── openai-chat-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "chat-tool" + │ "operation": "chat-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream" + │ "operation": "stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 │ } ├── openai-stream-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-with-response" + │ "operation": "stream-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-stream-tool-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-tool" + │ "operation": "stream-tool", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } │ } │ ] │ metadata: { + │ "max_tokens": 64, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "tool_choice": { + │ "function": { + │ "name": "get_weather" + │ }, + │ "type": "function" + │ }, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the weather for a location", + │ "name": "get_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "The location to get weather for", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 77, + │ "time_to_first_token": 0, + │ "tokens": 84 │ } ├── openai-stream-fixture-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "stream-fixture" + │ "operation": "stream-fixture", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with a refusal stream fixture.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "parse" + │ "operation": "parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "What is 2 + 2?", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [ - │ "answer" - │ ], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 │ } ├── openai-sync-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "sync-stream" + │ "operation": "sync-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Chat Completion [llm] │ input: [ │ { - │ "content_kind": "string", + │ "content": "Reply with exactly SYNC STREAM.", │ "role": "user" │ } │ ] │ output: [ │ { - │ "json_keys": [], - │ "role": "assistant" + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } │ } │ ] │ metadata: { + │ "max_tokens": 24, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "provider": "openai", + │ "temperature": 0 │ } │ metrics: { - │ "has_time_to_first_token": true + │ "time_to_first_token": 0 │ } ├── openai-embeddings-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "embeddings" + │ "operation": "embeddings", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Embedding [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Paris" │ output: { │ "embedding_length": 1536 │ } @@ -377,279 +710,891 @@ span_tree: │ "provider": "openai" │ } │ metrics: { - │ "has_time_to_first_token": false + │ "prompt_tokens": 1, + │ "tokens": 1 │ } ├── openai-moderations-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "moderations" + │ "operation": "moderations", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── Moderation [llm] - │ input: { - │ "kind": "text" - │ } - │ output: { - │ "flagged_count": 0, - │ "result_count": 1 - │ } + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] │ metadata: { │ "model": "omni-moderation-2024-09-26", │ "provider": "openai" │ } - │ metrics: { - │ "has_time_to_first_token": false + ├── openai-batch-operation + │ metadata: { + │ "operation": "batch", + │ "testRunId": "" + │ } + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 + │ } + │ └── openai.batch [task] + │ metadata: { + │ "provider": "openai" + │ } + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly ALPHA.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "ALPHA", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 9, + │ │ "tokens": 10 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ ├── Chat Completion [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply with exactly BRAVO.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finish_reason": "stop", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "BRAVO", + │ │ "role": "assistant" + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_tokens": 8, + │ │ "tokens": 9 + │ │ } + │ │ context: { + │ │ "caller_filename": "", + │ │ "caller_functionname": "", + │ │ "caller_lineno": 0 + │ │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly CHARLIE.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" + │ } + │ error: "Batch fixture request failed" + │ context: { + │ "caller_filename": "", + │ "caller_functionname": "", + │ "caller_lineno": 0 + │ } ├── openai-responses-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses" + │ "operation": "responses", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARIS." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-with-response-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-with-response" + │ "operation": "responses-with-response", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 2 + 2? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-create-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-create-stream" + │ "operation": "responses-create-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly RESPONSE STREAM." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 │ } ├── openai-responses-stream-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream" + │ "operation": "responses-stream", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 6 x 6? Reply with just the number." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 │ } ├── openai-responses-stream-partial-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-stream-partial" + │ "operation": "responses-stream-partial", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.create [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "Reply with exactly PARTIAL." │ output: [ │ { - │ "content_types": [ - │ "output_text" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } │ ], - │ "json_keys": [], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 │ } ├── openai-responses-parse-operation - │ input: { - │ "kind": "undefined" - │ } - │ output: null │ metadata: { - │ "operation": "responses-parse" + │ "operation": "responses-parse", + │ "testRunId": "" │ } - │ metrics: { - │ "has_time_to_first_token": false + │ context: { + │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", + │ "caller_functionname": "runOperation", + │ "caller_lineno": 0 │ } │ └── openai.responses.parse [llm] - │ input: { - │ "kind": "text" - │ } + │ input: "What is 20 + 4?" │ output: [ │ { - │ "content_types": [ - │ "output_text" - │ ], - │ "json_keys": [ - │ "reasoning", - │ "value" + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "parsed": { + │ "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + │ "value": 24 + │ }, + │ "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "type": "output_text" + │ } │ ], + │ "id": "", │ "role": "assistant", │ "status": "completed", │ "type": "message" │ } │ ] │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": null, + │ "max_tool_calls": null, + │ "metadata": {}, │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" + │ "moderation": null, + │ "object": "response", + │ "output_parsed": { + │ "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + │ "value": 24 + │ }, + │ "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "context": null, + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "description": null, + │ "name": "NumberAnswer", + │ "schema": { + │ "additionalProperties": false, + │ "properties": { + │ "reasoning": { + │ "type": "string" + │ }, + │ "value": { + │ "type": "integer" + │ } + │ }, + │ "required": [ + │ "value", + │ "reasoning" + │ ], + │ "type": "object" + │ }, + │ "strict": true, + │ "type": "json_schema" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tool_usage": { + │ "image_gen": { + │ "input_tokens": 0, + │ "input_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "output_tokens": 0, + │ "output_tokens_details": { + │ "image_tokens": 0, + │ "text_tokens": 0 + │ }, + │ "total_tokens": 0 + │ }, + │ "web_search": { + │ "num_requests": 0 + │ } + │ }, + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null │ } │ metrics: { - │ "has_time_to_first_token": true + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 30, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 46, + │ "time_to_first_token": 0, + │ "tokens": 76 │ } └── openai-responses-compact-operation - input: { - "kind": "undefined" - } - output: null metadata: { - "operation": "responses-compact" + "operation": "responses-compact", + "testRunId": "" } - metrics: { - "has_time_to_first_token": false + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runOperation", + "caller_lineno": 0 } └── openai.responses.compact [llm] input: [ { - "content_kind": "blocks", + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], "role": "user" }, { - "content_kind": "blocks", + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], "role": "assistant" } ] output: [ { - "content_types": [ - "input_text" + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } ], - "json_keys": [], + "id": "", "role": "user", "status": "completed", "type": "message" }, { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, + "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "id": "", "type": "compaction" } ] metadata: { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", "provider": "openai" } metrics: { - "has_time_to_first_token": true + "completion_reasoning_tokens": 0, + "completion_tokens": 220, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 353 } diff --git a/e2e/scenarios/openai-instrumentation/assertions.ts b/e2e/scenarios/openai-instrumentation/assertions.ts index f9684e4f2..5f7439e41 100644 --- a/e2e/scenarios/openai-instrumentation/assertions.ts +++ b/e2e/scenarios/openai-instrumentation/assertions.ts @@ -2,7 +2,6 @@ import { existsSync } from "node:fs"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; import { beforeAll, describe, expect, test } from "vitest"; -import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { @@ -45,7 +44,9 @@ type RelevantEvent = { type OperationSpec = { childNames: readonly string[]; + nestedChildNames?: readonly string[]; expectsOutput: boolean; + expectsModel?: boolean; expectsTimeToFirstToken: boolean; minOpenAIMajorVersion?: number; name: string; @@ -55,6 +56,12 @@ type OperationSpec = { validate?: (span: CapturedLogEvent | undefined) => void; }; +const EXPECTED_BATCH_OUTPUTS = new Map([ + ["Reply with exactly ALPHA.", "ALPHA"], + ["Reply with exactly BRAVO.", "BRAVO"], + ["Reply with exactly CHARLIE.", undefined], +]); + function asRecord(value: unknown): Record | undefined { return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) @@ -233,6 +240,20 @@ const OPERATION_SPECS: readonly OperationSpec[] = [ expect(Array.isArray(span?.output)).toBe(true); }, }, + { + childNames: ["openai.batch"], + nestedChildNames: ["Chat Completion"], + expectsModel: false, + expectsOutput: false, + expectsTimeToFirstToken: false, + name: "openai-batch-operation", + operation: "batch", + testName: "captures resumable OpenAI Batch task and LLM spans", + validate: (span) => { + expect(span?.input).toBeUndefined(); + expect(span?.output).toBeUndefined(); + }, + }, { childNames: ["openai.responses.create"], expectsOutput: true, @@ -320,245 +341,22 @@ function getOperationSpecs(version: string): OperationSpec[] { ); } -function pickMetadata( - metadata: Record | undefined, - keys: string[], -): Json { - if (!metadata) { - return null; - } - - const picked = Object.fromEntries( - keys.flatMap((key) => { - const value = metadata[key]; - if (value === undefined) { - return []; - } - - return [ - [ - key, - key === "openaiSdkVersion" && typeof value === "string" - ? "" - : (value as Json), - ], - ]; - }), - ); - - return Object.keys(picked).length > 0 ? (picked as Json) : null; -} - -function isRecord(value: Json | undefined): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function summarizeMetricPresence(metrics: Json): Json { - if (!isRecord(metrics)) { - return null; - } - - return { - has_time_to_first_token: typeof metrics.time_to_first_token === "number", - } satisfies Json; -} - -function jsonKeysFromText(value: unknown): string[] { - if (typeof value !== "string") { - return []; - } - - try { - const parsed = JSON.parse(value) as unknown; - if ( - typeof parsed !== "object" || - parsed === null || - Array.isArray(parsed) - ) { - return []; - } - - return Object.keys(parsed).sort(); - } catch { - return []; - } -} - -function summarizeInput(input: Json): Json { - if (typeof input === "string") { - return { - kind: "text", - } satisfies Json; - } - - if (!Array.isArray(input)) { - return input === null ? null : ({ kind: typeof input } satisfies Json); - } - - return input.map((message) => { - if (!isRecord(message as Json)) { - return { kind: typeof message } satisfies Json; - } - - return { - content_kind: Array.isArray(message.content) - ? "blocks" - : typeof message.content, - role: message.role ?? null, - } satisfies Json; - }) satisfies Json; -} - -function summarizeChatOutput(output: Json): Json { - if (!Array.isArray(output)) { - return null; - } - - return output.map((choice) => { - if (!isRecord(choice as Json) || !isRecord(choice.message as Json)) { - return null; - } - - return { - json_keys: jsonKeysFromText(choice.message.content), - role: choice.message.role ?? null, - } satisfies Json; - }) satisfies Json; -} - -function summarizeResponsesOutput( - output: Json, - options?: { - // For normalization across SDK versions - dropEmptyOutputTextMessages?: boolean; - }, -): Json { - if (!Array.isArray(output)) { - return null; - } - - const summaries = output.map((item) => { - if (!isRecord(item as Json)) { - return null; - } - - const content = Array.isArray(item.content) ? item.content : []; - const contentTypes = content.flatMap((entry) => - isRecord(entry as Json) && typeof entry.type === "string" - ? [entry.type] - : [], - ); - const jsonKeys = content.flatMap((entry) => - isRecord(entry as Json) ? jsonKeysFromText(entry.text) : [], - ); - - return { - content_types: contentTypes, - json_keys: [...new Set(jsonKeys)].sort(), - role: item.role ?? null, - status: item.status ?? null, - type: item.type ?? null, - } satisfies Json; - }); - - const filtered = options?.dropEmptyOutputTextMessages - ? summaries.filter((item) => { - if (!isRecord(item as Json)) { - return true; - } - - return !( - item.role === "assistant" && - item.status === "completed" && - item.type === "message" && - Array.isArray(item.content_types) && - item.content_types.length === 1 && - item.content_types[0] === "output_text" && - Array.isArray(item.json_keys) && - item.json_keys.length === 0 - ); - }) - : summaries; - - // Deduplicate identical items — the Responses API occasionally returns - // duplicate output entries (e.g., two identical "message" items when - // streaming), which would cause non-deterministic snapshot failures. - const seen = new Set(); - const deduped: Json[] = []; - - for (const summarized of filtered) { - const key = JSON.stringify(summarized); - if (!seen.has(key)) { - seen.add(key); - deduped.push(summarized); +function findOpenAISpan( + events: CapturedLogEvent[], + parentId: string | undefined, + names: readonly string[], +) { + for (const name of names) { + const spans = findChildSpans(events, name, parentId); + if (spans.length > 0) { + return spans.find((span) => span.output !== undefined) ?? spans.at(-1); } } - return deduped; -} - -function summarizeOutput(name: string, output: Json): Json { - if (name === "Chat Completion") { - return summarizeChatOutput(output); - } - - if (name === "Embedding") { - return isRecord(output) - ? ({ - embedding_length: output.embedding_length ?? null, - } satisfies Json) - : null; - } - - if (name === "Moderation") { - return Array.isArray(output) - ? ({ - flagged_count: output.filter( - (entry) => isRecord(entry as Json) && entry.flagged === true, - ).length, - result_count: output.length, - } satisfies Json) - : null; - } - - if ( - name === "openai.responses.create" || - name === "openai.responses.compact" - ) { - return summarizeResponsesOutput(output); - } - - if (name === "openai.responses.parse") { - return summarizeResponsesOutput(output, { - dropEmptyOutputTextMessages: true, - }); - } - - return output === null || output === undefined - ? null - : ({ kind: typeof output } satisfies Json); -} - -function summarizeOpenAIPayload( - event: CapturedLogEvent, - summaryName: string | undefined, -): Json { - const name = summaryName ?? event.span.name ?? ""; - - return { - input: summarizeInput(event.input as Json), - metadata: pickMetadata( - event.row.metadata as Record | undefined, - ["model", "openaiSdkVersion", "operation", "provider", "scenario"], - ), - metrics: summarizeMetricPresence(event.metrics as Json), - name: name || null, - output: summarizeOutput(name, event.output as Json), - type: event.span.type ?? null, - } satisfies Json; + return undefined; } -function findOpenAISpan( +function findOpenAISpans( events: CapturedLogEvent[], parentId: string | undefined, names: readonly string[], @@ -566,11 +364,17 @@ function findOpenAISpan( for (const name of names) { const spans = findChildSpans(events, name, parentId); if (spans.length > 0) { - return spans.find((span) => span.output !== undefined) ?? spans.at(-1); + return spans; } } - return undefined; + return []; +} + +function batchPrompt(span: CapturedLogEvent): string { + const firstMessage = Array.isArray(span.input) ? span.input[0] : undefined; + const content = asRecord(firstMessage)?.content; + return typeof content === "string" ? content : ""; } function buildRelevantEvents( @@ -584,10 +388,27 @@ function buildRelevantEvents( for (const spec of operationSpecs) { const operation = findLatestSpan(events, spec.name)!; relevantEvents.push({ event: operation }); + const providerSpan = findOpenAISpan( + events, + operation.span.id, + spec.childNames, + )!; relevantEvents.push({ - event: findOpenAISpan(events, operation.span.id, spec.childNames)!, + event: providerSpan, summaryName: spec.childNames[0], }); + if (spec.nestedChildNames) { + relevantEvents.push( + ...findOpenAISpans(events, providerSpan.span.id, spec.nestedChildNames) + .sort((left, right) => + batchPrompt(left).localeCompare(batchPrompt(right)), + ) + .map((event) => ({ + event, + summaryName: spec.nestedChildNames?.[0], + })), + ); + } } return relevantEvents; @@ -599,22 +420,13 @@ function buildSpanTree( ): SpanTreeEntry[] { return buildRelevantEvents(events, operationSpecs).map( ({ event, summaryName }) => { - const summary = summarizeOpenAIPayload(event, summaryName) as Record< - string, - Json - >; - const { name: _name, type: _type, ...fields } = summary; - return { event, fields: { - span_attributes: spanTreeFields(event).span_attributes, - ...fields, + ...spanTreeFields(event), + context: event.context, }, - name: - typeof summary.name === "string" - ? summary.name - : (summaryName ?? event.span.name), + name: summaryName ?? event.span.name, }; }, ); @@ -688,6 +500,17 @@ export function defineOpenAIInstrumentationAssertions(options: { spec.childNames, ); expect(spanInstrumentationName(span)).toBe("openai"); + if (spec.nestedChildNames) { + const nested = findOpenAISpans( + events, + span?.span.id, + spec.nestedChildNames, + ); + expect(nested).toHaveLength(EXPECTED_BATCH_OUTPUTS.size); + for (const child of nested) { + expect(spanInstrumentationName(child)).toBe("openai"); + } + } } }); @@ -724,9 +547,12 @@ export function defineOpenAIInstrumentationAssertions(options: { expect(span?.row.metadata).toMatchObject({ provider: "openai", }); - expect( - typeof (span?.row.metadata as { model?: unknown } | undefined)?.model, - ).toBe("string"); + if (spec.expectsModel !== false) { + expect( + typeof (span?.row.metadata as { model?: unknown } | undefined) + ?.model, + ).toBe("string"); + } if (spec.expectsOutput) { expect(span?.output).toBeDefined(); @@ -744,6 +570,41 @@ export function defineOpenAIInstrumentationAssertions(options: { ); } + if (spec.nestedChildNames) { + const nested = findOpenAISpans( + events, + span?.span.id, + spec.nestedChildNames, + ); + expect(nested).toHaveLength(EXPECTED_BATCH_OUTPUTS.size); + expect(span?.span.parentIds).toEqual([operation?.span.id ?? ""]); + expect(nested.map(batchPrompt).sort()).toEqual( + [...EXPECTED_BATCH_OUTPUTS.keys()].sort(), + ); + for (const child of nested) { + const prompt = batchPrompt(child); + const firstChoice = Array.isArray(child.output) + ? asRecord(child.output[0]) + : undefined; + const message = asRecord(firstChoice?.message); + + expect(child.span.parentIds).toEqual([span?.span.id ?? ""]); + expect(child.row.metadata).toMatchObject({ + model: expect.any(String), + provider: "openai", + }); + const expectedOutput = EXPECTED_BATCH_OUTPUTS.get(prompt); + if (expectedOutput) { + expect(message?.content).toBe(expectedOutput); + expect(child.row.error).toBeUndefined(); + } else { + expect(child.output).toBeUndefined(); + expect(child.row.error).toContain("Batch fixture request failed"); + } + expect(child.metrics?.time_to_first_token).toBeUndefined(); + } + } + spec.validate?.(span); }); } diff --git a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs index 076c39492..d17abf4fb 100644 --- a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs @@ -7,6 +7,7 @@ import { runOperation, runTracedScenario, } from "../../helpers/provider-runtime.mjs"; +import { completeOpenAIBatch, createOpenAIBatch } from "braintrust"; const OPENAI_MODEL = "gpt-4o-mini-2024-07-18"; const EMBEDDING_MODEL = "text-embedding-3-small"; @@ -104,6 +105,40 @@ function createMockStreamingClient(options) { : baseClient; } +function createMockBatchClient(options) { + const baseClient = new options.OpenAI({ + apiKey: process.env.OPENAI_API_KEY ?? "test-openai-key", + baseURL: "https://example.test/v1", + fetch: async (_url, init) => { + const params = JSON.parse(String(init?.body)); + return new Response( + JSON.stringify({ + id: "batch_e2e_fixture", + object: "batch", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + completion_window: params.completion_window, + status: "validating", + created_at: 1_740_000_000, + metadata: params.metadata, + request_counts: { completed: 0, failed: 0, total: 0 }, + }), + { + headers: { + "content-type": "application/json", + "x-request-id": "req_batch_e2e_fixture", + }, + status: 200, + }, + ); + }, + }); + + return options.decorateClient + ? options.decorateClient(baseClient) + : baseClient; +} + export async function runOpenAIInstrumentationScenario(options) { const baseClient = new options.OpenAI({ apiKey: process.env.OPENAI_API_KEY, @@ -113,6 +148,7 @@ export async function runOpenAIInstrumentationScenario(options) { ? options.decorateClient(baseClient) : baseClient; const streamFixtureClient = createMockStreamingClient(options); + const batchFixtureClient = createMockBatchClient(options); const openAIMajorVersion = parseMajorVersion(options.openaiSdkVersion); const shouldCheckPrivateFieldMethods = typeof options.decorateClient === "function" && @@ -582,6 +618,116 @@ export async function runOpenAIInstrumentationScenario(options) { }, ); } + + await runOperation("openai-batch-operation", "batch", async () => { + const batchItems = [ + { + customId: "batch_chat_alpha", + prompt: "Reply with exactly ALPHA.", + response: "ALPHA", + }, + { + customId: "batch_chat_bravo", + prompt: "Reply with exactly BRAVO.", + response: "BRAVO", + }, + { + customId: "batch_chat_charlie", + prompt: "Reply with exactly CHARLIE.", + response: "CHARLIE", + }, + ]; + const input = batchItems + .map((item) => + JSON.stringify({ + custom_id: item.customId, + method: "POST", + url: "/v1/chat/completions", + body: { + model: OPENAI_MODEL, + messages: [{ role: "user", content: item.prompt }], + }, + }), + ) + .join("\n"); + const created = await createOpenAIBatch({ + client: batchFixtureClient, + inputFile: { id: "file_batch_e2e_fixture" }, + input, + params: { + completion_window: "24h", + endpoint: "/v1/chat/completions", + }, + }); + + const completedAt = Date.now() / 1000 + 60; + const completed = { + ...created, + status: "completed", + in_progress_at: 1_740_000_010, + completed_at: completedAt, + request_counts: { completed: 2, failed: 1, total: 3 }, + }; + // Start both content requests before awaiting either one. These promises + // model the APIPromise values returned by files.content(). + const outputFile = Promise.resolve( + new Response( + [batchItems[1], batchItems[0]] + .map((item, index) => + JSON.stringify({ + custom_id: item.customId, + response: { + status_code: 200, + body: { + choices: [ + { + index: 0, + finish_reason: "stop", + message: { + role: "assistant", + content: item.response, + }, + }, + ], + usage: { + prompt_tokens: 8 + index, + completion_tokens: 1, + total_tokens: 9 + index, + }, + }, + }, + }), + ) + .join("\n"), + ), + ); + const errorFile = Promise.resolve( + new Response( + JSON.stringify({ + custom_id: batchItems[2].customId, + error: { + code: "fixture_error", + message: "Batch fixture request failed", + }, + }), + ), + ); + await completeOpenAIBatch({ + batch: completed, + input, + // Batch results are not guaranteed to preserve input order. + outputFile, + errorFile, + webhook: { + type: "batch.completed", + created_at: completedAt, + data: { id: completed.id }, + }, + }); + if ((await outputFile).bodyUsed || (await errorFile).bodyUsed) { + throw new Error("Expected batch result responses to remain unread"); + } + }); }, metadata: { openaiSdkVersion: options.openaiSdkVersion, diff --git a/js/src/exports.ts b/js/src/exports.ts index 2f4332842..ccca0f2d6 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -129,6 +129,8 @@ export { export { registerSandbox } from "./sandbox"; +export { completeOpenAIBatch, createOpenAIBatch } from "./openai-batch"; + // Internal isomorph layer for platform-specific implementations import _internalIso from "./isomorph"; export { _internalIso }; diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index 4ed98d53e..071869f6a 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -406,9 +406,9 @@ export class BraintrustPlugin extends BasePlugin { } } -// Re-export utility functions from OpenAIPlugin for backward compatibility +// Re-export OpenAI instrumentation utilities from their canonical modules. export { parseMetricsFromUsage, - processImagesInOutput, aggregateChatCompletionChunks, } from "./plugins/openai-plugin"; +export { processImagesInOutput } from "./plugins/openai-span-data"; diff --git a/js/src/instrumentation/plugins/openai-batch-constants.ts b/js/src/instrumentation/plugins/openai-batch-constants.ts new file mode 100644 index 000000000..5451014b1 --- /dev/null +++ b/js/src/instrumentation/plugins/openai-batch-constants.ts @@ -0,0 +1 @@ +export const BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY = "braintrust.batch_context"; diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts new file mode 100644 index 000000000..da44afcc6 --- /dev/null +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts @@ -0,0 +1,722 @@ +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + _exportsForTestingOnly, + _internalGetGlobalState, + initLogger, + startSpan, + withCurrent, +} from "../../logger"; +import { configureNode } from "../../node/config"; +import { completeOpenAIBatch, createOpenAIBatch } from "../../openai-batch"; +import type { + OpenAIBatchCreateInputParams, + OpenAIBatchJSONL, + OpenAIFileLike, +} from "../../openai-batch-types"; +import { BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY } from "./openai-batch-constants"; +import { OpenAIPlugin } from "./openai-plugin"; +import { SpanComponentsV4 } from "../../../util/span_identifier_v4"; + +try { + configureNode(); +} catch { + // Best-effort initialization for test environments. +} + +const chatInput = [ + { + custom_id: "ok", + method: "POST", + url: "/v1/chat/completions", + body: { + model: "gpt-test", + messages: [{ role: "user", content: "hi" }], + }, + }, + { + custom_id: "bad", + method: "POST", + url: "/v1/chat/completions", + body: { + model: "gpt-test", + messages: [{ role: "user", content: "fail" }], + }, + }, +]; + +async function createTestBatch(args: { + input: OpenAIBatchJSONL; + inputFile: OpenAIFileLike; + params: OpenAIBatchCreateInputParams; +}) { + return await createOpenAIBatch({ + ...args, + client: { + batches: { + create: async (params) => ({ + id: `batch_${args.inputFile.id}`, + ...params, + status: "validating", + }), + }, + }, + }); +} + +function chunkedJSONLResponse( + records: unknown[], + beforeFirstChunk?: () => Promise, +): Response { + const bytes = new TextEncoder().encode( + records.map((record) => JSON.stringify(record)).join("\n"), + ); + let offset = 0; + return new Response( + new ReadableStream( + { + async pull(controller) { + if (offset === 0) { + await beforeFirstChunk?.(); + } + if (offset < bytes.length) { + controller.enqueue(bytes.slice(offset, offset + 7)); + offset += 7; + } else { + controller.close(); + } + }, + }, + { highWaterMark: 0 }, + ), + ); +} + +describe("OpenAI Batch instrumentation", () => { + let backgroundLogger: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger + >; + let plugin: OpenAIPlugin; + + beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + _internalGetGlobalState()._internalSetTraceContextSigningSecret( + "openai-batch-test-secret", + ); + }); + + beforeEach(() => { + backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ + projectName: "openai-batch-instrumentation.test.ts", + projectId: "test-project-id", + }); + plugin = new OpenAIPlugin(); + plugin.enable(); + }); + + afterEach(() => { + plugin.disable(); + _exportsForTestingOnly.clearTestBackgroundLogger(); + }); + + it("creates a batch and starts deterministic pending spans", async () => { + const inputFile = { id: "file_input", filename: "batch.jsonl" }; + const input = chatInput.map((record) => ({ + ...record, + body: { ...record.body, user: "not-span-metadata" }, + })); + const sourceParams = { + endpoint: "/v1/chat/completions" as const, + completion_window: "24h" as const, + metadata: { user: "value" }, + }; + const params = await createTestBatch({ + inputFile, + input: input.map((record) => JSON.stringify(record)).join("\n"), + params: sourceParams, + }); + + expect(params).toMatchObject({ + input_file_id: "file_input", + endpoint: "/v1/chat/completions", + completion_window: "24h", + metadata: { + user: "value", + [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: expect.any(String), + }, + }); + expect(sourceParams.metadata).toEqual({ user: "value" }); + expect(inputFile).toEqual({ id: "file_input", filename: "batch.jsonl" }); + + const rows = (await backgroundLogger.drain()) as Array>; + expect(rows).toHaveLength(3); + const task = rows.find( + (row) => row.span_attributes?.name === "openai.batch", + ); + const children = rows.filter( + (row) => row.span_attributes?.name === "Chat Completion", + ); + expect(task).toMatchObject({ + metadata: { provider: "openai" }, + span_attributes: { type: "task" }, + }); + expect(task?.input).toBeUndefined(); + expect(task?.metrics).not.toHaveProperty("end"); + expect(task?.span_parents).toEqual([]); + expect(children).toHaveLength(2); + expect(children.map((row) => row.input?.[0]?.content).sort()).toEqual([ + "fail", + "hi", + ]); + expect(children[0]?.metrics).not.toHaveProperty("end"); + expect( + children.find((row) => row.input?.[0]?.content === "hi"), + ).toMatchObject({ + input: [{ role: "user", content: "hi" }], + metadata: { model: "gpt-test", provider: "openai" }, + }); + expect( + children.find((row) => row.input?.[0]?.content === "hi")?.metadata, + ).not.toHaveProperty("user"); + }); + + it("does not start spans when OpenAI returns a different batch context", async () => { + const batch = await createOpenAIBatch({ + inputFile: { id: "file_mismatched_response" }, + input: [chatInput[0]], + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + client: { + batches: { + create: async (params) => ({ + ...params, + id: "batch_mismatched_response", + endpoint: "/v1/responses", + status: "validating", + }), + }, + }, + }); + + expect(batch.endpoint).toBe("/v1/responses"); + expect(await backgroundLogger.drain()).toEqual([]); + }); + + it("keeps separate batch operations distinct when they reuse an input file", async () => { + const args = { + inputFile: { id: "file_reused" }, + input: [chatInput[0]], + params: { + endpoint: "/v1/chat/completions" as const, + completion_window: "24h" as const, + }, + }; + + const first = await createTestBatch(args); + const firstRows = (await backgroundLogger.drain()) as Array< + Record + >; + const second = await createTestBatch(args); + const secondRows = (await backgroundLogger.drain()) as Array< + Record + >; + + expect(first.metadata?.[BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]).not.toBe( + second.metadata?.[BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY], + ); + expect(firstRows).toHaveLength(2); + expect(secondRows).toHaveLength(2); + const firstIds = new Set(firstRows.map((row) => row.id)); + expect(secondRows.some((row) => firstIds.has(row.id))).toBe(false); + }); + + it("signs and verifies batch context with WebCrypto", async () => { + const params = await createTestBatch({ + inputFile: { id: "file_webcrypto" }, + input: [chatInput[0]], + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }); + expect(params.metadata).toHaveProperty(BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY); + await backgroundLogger.drain(); + + await completeOpenAIBatch({ + batch: { + id: "batch_webcrypto", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + status: "completed", + completed_at: 200, + metadata: params.metadata, + }, + input: [chatInput[0]], + outputFile: [ + { + custom_id: "ok", + response: { + status_code: 200, + body: { choices: [], usage: { total_tokens: 0 } }, + }, + }, + ], + }); + expect(await backgroundLogger.drain()).toHaveLength(2); + }); + + it("does not send propagated span event data to OpenAI", async () => { + const parent = startSpan({ + name: "sensitive-parent", + propagatedEvent: { metadata: { secret: "do-not-send" } }, + }); + const batch = await withCurrent(parent, () => + createTestBatch({ + inputFile: { id: "file_minimal_parent" }, + input: [chatInput[0]], + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }), + ); + parent.end(); + + const serialized = JSON.parse( + batch.metadata?.[BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY] ?? "null", + ) as { parent?: unknown; version?: unknown } | null; + expect(serialized?.version).toBe(1); + expect(typeof serialized?.parent).toBe("string"); + const exported = SpanComponentsV4.fromStr(serialized?.parent as string); + expect(exported.data.propagated_event).toBeUndefined(); + }); + + it("handles file promise rejections on early completion paths", async () => { + const batch = { + id: "batch_in_progress", + endpoint: "/v1/chat/completions", + input_file_id: "file_input", + status: "in_progress", + }; + + await expect( + completeOpenAIBatch({ + batch, + input: chatInput, + outputFile: Promise.reject(new Error("output unavailable")), + errorFile: Promise.reject(new Error("errors unavailable")), + }), + ).resolves.toBe(batch); + await new Promise((resolve) => queueMicrotask(resolve)); + expect(await backgroundLogger.drain()).toEqual([]); + }); + + it("completes successful and failed spans from resumable files", async () => { + const params = await createTestBatch({ + inputFile: { id: "file_input" }, + input: chatInput, + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }); + const pendingRows = (await backgroundLogger.drain()) as Array< + Record + >; + const pendingIds = pendingRows.map((row) => row.id).sort(); + const batch = { + id: "batch_complete", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + completion_window: params.completion_window, + status: "completed", + completed_at: 140, + request_counts: { completed: 1, failed: 1, total: 2 }, + metadata: params.metadata, + }; + const outputRecords = [ + { + custom_id: "ok", + response: { + status_code: 200, + body: { + choices: [ + { + index: 0, + message: { role: "assistant", content: "hello 👋" }, + }, + ], + model: "gpt-resolved", + usage: { prompt_tokens: 2, completion_tokens: 3, total_tokens: 5 }, + }, + }, + }, + ]; + const errorRecords = [ + { custom_id: "bad", error: { code: "server_error", message: "failed" } }, + ]; + let readersStarted = 0; + let releaseReaders: (() => void) | undefined; + const bothReadersStarted = new Promise((resolve) => { + releaseReaders = resolve; + }); + const waitForBothReaders = async () => { + readersStarted++; + if (readersStarted === 2) { + releaseReaders?.(); + } + await bothReadersStarted; + }; + const outputResponse = chunkedJSONLResponse( + outputRecords, + waitForBothReaders, + ); + const errorResponse = chunkedJSONLResponse( + errorRecords, + waitForBothReaders, + ); + const completedAt = Date.now() / 1000 + 60; + batch.completed_at = completedAt; + + await expect( + completeOpenAIBatch({ + batch, + input: chatInput, + outputFile: Promise.resolve(outputResponse), + errorFile: Promise.resolve(errorResponse), + }), + ).resolves.toBe(batch); + expect(readersStarted).toBe(2); + expect(outputResponse.bodyUsed).toBe(false); + expect(errorResponse.bodyUsed).toBe(false); + const rows = (await backgroundLogger.drain()) as Array>; + expect(rows).toHaveLength(3); + expect(rows.map((row) => row.id).sort()).toEqual(pendingIds); + const task = rows.find( + (row) => row.span_attributes?.name === "openai.batch", + ); + const children = rows.filter( + (row) => row.span_attributes?.name === "Chat Completion", + ); + expect(task).toMatchObject({ + metrics: { end: completedAt }, + }); + expect(task?.output).toBeUndefined(); + expect(children.find((row) => row.output !== undefined)).toMatchObject({ + metadata: { model: "gpt-resolved", provider: "openai" }, + metrics: { + completion_tokens: 3, + end: completedAt, + prompt_tokens: 2, + tokens: 5, + }, + output: [ + { index: 0, message: { role: "assistant", content: "hello 👋" } }, + ], + }); + expect(children.find((row) => row.error !== undefined)?.error).toContain( + "failed", + ); + + await completeOpenAIBatch({ + batch, + input: chatInput, + outputFile: Promise.resolve(chunkedJSONLResponse(outputRecords)), + errorFile: Promise.resolve(chunkedJSONLResponse(errorRecords)), + }); + const replayRows = (await backgroundLogger.drain()) as Array< + Record + >; + expect(replayRows.map((row) => row.id).sort()).toEqual(pendingIds); + }); + + it("leaves a completed batch pending until all results are available", async () => { + const batch = await createTestBatch({ + inputFile: { id: "file_incomplete" }, + input: chatInput, + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }); + await backgroundLogger.drain(); + + await completeOpenAIBatch({ + batch: { + ...batch, + status: "completed", + completed_at: 400, + request_counts: { completed: 1, failed: 1, total: 2 }, + }, + input: chatInput, + outputFile: [ + { + custom_id: "ok", + response: { + status_code: 200, + body: { choices: [], model: "gpt-resolved" }, + }, + }, + ], + }); + + const rows = (await backgroundLogger.drain()) as Array>; + const task = rows.find( + (row) => row.span_attributes?.name === "openai.batch", + ); + expect(task?.metrics).not.toHaveProperty("end"); + expect(task?.output).toBeUndefined(); + }); + + it("rejects completion input that does not match the signed batch input", async () => { + const params = await createTestBatch({ + inputFile: { id: "file_mismatched" }, + input: chatInput, + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }); + await backgroundLogger.drain(); + + await completeOpenAIBatch({ + batch: { + id: "batch_mismatched", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + status: "completed", + completed_at: 400, + request_counts: { completed: 1, failed: 1, total: 2 }, + metadata: params.metadata, + }, + input: chatInput.map((record) => ({ + ...record, + body: { + ...record.body, + messages: [{ role: "user", content: `changed-${record.custom_id}` }], + }, + })), + outputFile: [ + { + custom_id: "ok", + response: { + status_code: 200, + body: { choices: [], model: "gpt-resolved" }, + }, + }, + ], + errorFile: [{ custom_id: "bad", error: { message: "expected failure" } }], + }); + + expect(await backgroundLogger.drain()).toEqual([]); + }); + + it("falls back from invalid provider terminal timestamps", async () => { + const params = await createTestBatch({ + inputFile: { id: "file_invalid_timestamp" }, + input: [chatInput[0]], + params: { + endpoint: "/v1/chat/completions", + completion_window: "24h", + }, + }); + await backgroundLogger.drain(); + const beforeCompletion = Date.now() / 1000; + + await completeOpenAIBatch({ + batch: { + id: "batch_invalid_timestamp", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + status: "completed", + completed_at: beforeCompletion - 1, + request_counts: { completed: 1, failed: 0, total: 1 }, + metadata: params.metadata, + }, + input: [chatInput[0]], + outputFile: [ + { + custom_id: "ok", + response: { + status_code: 200, + body: { choices: [], model: "gpt-resolved" }, + }, + }, + ], + webhook: { + type: "batch.completed", + created_at: beforeCompletion - 1, + data: { id: "batch_invalid_timestamp" }, + }, + }); + + const rows = (await backgroundLogger.drain()) as Array>; + for (const row of rows) { + expect(row.metrics?.end).toEqual(expect.any(Number)); + expect(row.metrics.end).toBeGreaterThanOrEqual(beforeCompletion); + } + }); + + it("maps Responses records from async input and validates webhook context", async () => { + async function* input() { + yield { + custom_id: "response_1", + method: "POST", + url: "/v1/responses", + body: { model: "gpt-test", input: "hello" }, + }; + } + const params = await createTestBatch({ + inputFile: { id: "file_responses" }, + input: input(), + params: { endpoint: "/v1/responses", completion_window: "24h" }, + }); + await backgroundLogger.drain(); + const batch = { + id: "batch_responses", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + status: "completed", + completed_at: 260, + request_counts: { completed: 1, failed: 0, total: 1 }, + metadata: params.metadata, + }; + const outputFile = [ + { + custom_id: "response_1", + response: { + status_code: 200, + body: { + id: "resp_1", + model: "gpt-test", + output: [{ type: "message", role: "assistant", content: [] }], + usage: { input_tokens: 4, output_tokens: 6, total_tokens: 10 }, + }, + }, + }, + ]; + const completedAt = Date.now() / 1000 + 60; + batch.completed_at = completedAt; + + await completeOpenAIBatch({ + batch, + input: input(), + outputFile, + webhook: { + type: "batch.completed", + created_at: 260, + data: { id: batch.id }, + }, + }); + const rows = (await backgroundLogger.drain()) as Array>; + const responseSpan = rows.find( + (row) => row.span_attributes?.name === "openai.responses.create", + ); + expect(responseSpan).toMatchObject({ + metadata: { + model: "gpt-test", + provider: "openai", + }, + metrics: { + completion_tokens: 6, + end: completedAt, + prompt_tokens: 4, + tokens: 10, + }, + output: [{ type: "message", role: "assistant", content: [] }], + }); + expect(responseSpan?.metrics).not.toHaveProperty("time_to_first_token"); + + await completeOpenAIBatch({ + batch, + input: input(), + outputFile, + webhook: { type: "batch.completed", data: { id: "another_batch" } }, + }); + expect(await backgroundLogger.drain()).toEqual([]); + }); + + it("ends a result span when output processing throws", async () => { + const params = await createTestBatch({ + inputFile: { id: "file_invalid_output" }, + input: [ + { + custom_id: "response_1", + method: "POST", + url: "/v1/responses", + body: { model: "gpt-test", input: "hello" }, + }, + ], + params: { endpoint: "/v1/responses", completion_window: "24h" }, + }); + await backgroundLogger.drain(); + + const output = new Proxy([], { + get(target, property, receiver) { + if (property === "map") { + throw new Error("could not process output"); + } + return Reflect.get(target, property, receiver); + }, + }); + const completedAt = Date.now() / 1000 + 60; + await completeOpenAIBatch({ + batch: { + id: "batch_invalid_output", + endpoint: params.endpoint, + input_file_id: params.input_file_id, + status: "completed", + completed_at: completedAt, + metadata: params.metadata, + }, + input: [ + { + custom_id: "response_1", + method: "POST", + url: "/v1/responses", + body: { model: "gpt-test", input: "hello" }, + }, + ], + outputFile: [ + { + custom_id: "response_1", + response: { + status_code: 200, + body: { output }, + }, + }, + ], + }); + + const rows = (await backgroundLogger.drain()) as Array>; + const child = rows.find( + (row) => row.span_attributes?.name === "openai.responses.create", + ); + expect(child?.error).toContain("could not process output"); + expect(child?.metrics?.end).toBe(completedAt); + }); + + it("leaves conflicting reserved metadata untouched", async () => { + let inputReads = 0; + async function* input() { + inputReads++; + yield* chatInput; + } + const params = await createTestBatch({ + inputFile: { id: "file_input" }, + input: input(), + params: { + endpoint: "/v1/responses", + completion_window: "24h", + metadata: { [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: "user-owned" }, + }, + }); + expect(params.metadata).toEqual({ + [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: "user-owned", + }); + expect(inputReads).toBe(0); + expect(await backgroundLogger.drain()).toEqual([]); + }); +}); diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts new file mode 100644 index 000000000..fcd81e434 --- /dev/null +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts @@ -0,0 +1,1120 @@ +import { debugLogger } from "../../debug-logger"; +import { + _internalGetGlobalState, + _internalStartSpanWithInitialMerge, + _internalStartSpanWithInitialMergeAndParentSpanIds, + getSpanParentObject, + NOOP_SPAN, + type Span, + withCurrent, +} from "../../logger"; +import { parseMetricsFromUsage } from "../../openai-utils"; +import { + INSTRUMENTATION_NAMES, + withSpanInstrumentationName, +} from "../../span-origin"; +import { getCurrentUnixTimestamp } from "../../util"; +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { + SpanComponentsV4, + type SpanComponentsV4Data, +} from "../../../util/span_identifier_v4"; +import type { + CompleteOpenAIBatchArgs, + CreateOpenAIBatchArgs, + OpenAIBatchCreateParams, + OpenAIBatchJSONL, + OpenAIBatchLike, +} from "../../openai-batch-types"; +import { BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY } from "./openai-batch-constants"; +import { openAIChannels } from "./openai-channels"; +import { + extractOpenAIBatchInput, + processImagesInOutput, +} from "./openai-span-data"; + +const SUPPORTED_ENDPOINTS = new Set(["/v1/chat/completions", "/v1/responses"]); +const TERMINAL_STATUSES = new Set([ + "completed", + "failed", + "expired", + "cancelled", +]); +const INPUT_DIGEST_CHUNK_SIZE = 1024 * 1024; + +type BatchContext = { + version: 1; + parent: string; + inputFileId: string; + inputDigest: string; + endpoint: string; + startTime: number; + operationNonce: string; +}; + +type SerializedBatchContext = Omit & { + signature: string; +}; + +type BatchResultRecord = { + value: Record; + source: "output" | "error"; +}; + +type BatchInputRecord = { + customId: string; + spanData?: ReturnType; +}; + +type OpenAIBatchResultFile = + CompleteOpenAIBatchArgs["outputFile"]; + +type PreparedBatch = { + context: BatchContext; + inputs: Map; + params: OpenAIBatchCreateParams; +}; + +function read(value: unknown, key: PropertyKey): unknown { + if (!isObject(value)) { + return undefined; + } + try { + return Reflect.get(value, key); + } catch { + return undefined; + } +} + +function isBatchRecordIterable( + value: unknown, +): value is Iterable | AsyncIterable { + if ( + value === null || + (typeof value !== "object" && typeof value !== "function") + ) { + return false; + } + try { + return ( + typeof Reflect.get(value, Symbol.iterator) === "function" || + typeof Reflect.get(value, Symbol.asyncIterator) === "function" + ); + } catch { + return false; + } +} + +function validCustomId(value: unknown): value is string { + return typeof value === "string" && value.length > 0 && value.length <= 64; +} + +function logBatchInstrumentationError(context: string, error: unknown): void { + debugLogger.debug(`OpenAI Batch instrumentation ${context}:`, error); +} + +function batchContextPayload(context: BatchContext): string { + return JSON.stringify([BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY, context]); +} + +async function importBatchSigningKey(secret: string) { + return await globalThis.crypto.subtle.importKey( + "raw", + new TextEncoder().encode(secret), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign", "verify"], + ); +} + +async function signBatchContext( + context: BatchContext, + secret: string, +): Promise { + try { + const signature = await globalThis.crypto.subtle.sign( + "HMAC", + await importBatchSigningKey(secret), + new TextEncoder().encode(batchContextPayload(context)), + ); + return digestHex(new Uint8Array(signature), 32); + } catch { + return undefined; + } +} + +function signatureBytes(value: string): Uint8Array | undefined { + if (!/^[0-9a-f]{64}$/.test(value)) { + return undefined; + } + return Uint8Array.from(value.match(/.{2}/g) ?? [], (byte) => + Number.parseInt(byte, 16), + ); +} + +async function verifyBatchContext( + context: BatchContext, + signature: string, +): Promise { + const secret = + _internalGetGlobalState()._internalGetTraceContextSigningSecret(); + if (!secret) { + return false; + } + const bytes = signatureBytes(signature); + if (!bytes) { + return false; + } + try { + return await globalThis.crypto.subtle.verify( + "HMAC", + await importBatchSigningKey(secret), + bytes, + new TextEncoder().encode(batchContextPayload(context)), + ); + } catch { + return false; + } +} + +async function parseBatchContext( + value: unknown, + inputFileId: string, + endpoint: string, +): Promise { + if (typeof value !== "string") { + return undefined; + } + + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + return undefined; + } + if ( + !isObject(parsed) || + parsed.version !== 1 || + typeof parsed.parent !== "string" || + typeof parsed.inputDigest !== "string" || + !/^[0-9a-f]{64}$/.test(parsed.inputDigest) || + typeof parsed.startTime !== "number" || + !Number.isFinite(parsed.startTime) || + typeof parsed.operationNonce !== "string" || + !/^[0-9a-f]{32}$/.test(parsed.operationNonce) || + typeof parsed.signature !== "string" + ) { + return undefined; + } + + const context: BatchContext = { + version: 1, + parent: parsed.parent, + inputFileId, + inputDigest: parsed.inputDigest, + endpoint, + startTime: parsed.startTime, + operationNonce: parsed.operationNonce, + }; + if (!(await verifyBatchContext(context, parsed.signature))) { + return undefined; + } + + try { + SpanComponentsV4.fromStr(context.parent); + } catch { + return undefined; + } + return context; +} + +async function exportParent(parent: ReturnType) { + if ("toStr" in parent && typeof parent.toStr === "function") { + return parent.toStr(); + } + return await parent.export(); +} + +async function exportMinimalParent( + parent: ReturnType, +): Promise { + const exported = await exportParent(parent); + if (!exported) { + return undefined; + } + const parsed = SpanComponentsV4.fromStr(exported).data; + const projectId = read(parsed.compute_object_metadata_args, "project_id"); + const projectName = read(parsed.compute_object_metadata_args, "project_name"); + const computeObjectMetadataArgs = + typeof projectId === "string" || typeof projectName === "string" + ? { + ...(typeof projectId === "string" ? { project_id: projectId } : {}), + ...(typeof projectName === "string" + ? { project_name: projectName } + : {}), + } + : undefined; + if (!parsed.object_id && !computeObjectMetadataArgs) { + return undefined; + } + + const routing = parsed.object_id + ? { object_id: parsed.object_id } + : { compute_object_metadata_args: computeObjectMetadataArgs ?? {} }; + // SpanComponents requires all three span identifiers together. The + // conditional below preserves that invariant, but TypeScript cannot infer + // it through the object spreads. + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return new SpanComponentsV4({ + object_type: parsed.object_type, + ...routing, + ...(parsed.row_id && parsed.span_id && parsed.root_span_id + ? { + row_id: parsed.row_id, + span_id: parsed.span_id, + root_span_id: parsed.root_span_id, + } + : {}), + } as SpanComponentsV4Data).toStr(); +} + +async function prepareCreateParams( + args: CreateOpenAIBatchArgs, + parent: ReturnType, + startTime: number, +): Promise { + const params: OpenAIBatchCreateParams = { + ...args.params, + input_file_id: args.inputFile.id, + }; + const endpoint = read(params, "endpoint"); + const inputFileId = read(params, "input_file_id"); + if ( + typeof endpoint !== "string" || + !SUPPORTED_ENDPOINTS.has(endpoint) || + typeof inputFileId !== "string" || + inputFileId.length === 0 + ) { + return undefined; + } + + const metadataValue = read(params, "metadata"); + if ( + metadataValue !== undefined && + metadataValue !== null && + (!isObject(metadataValue) || Array.isArray(metadataValue)) + ) { + logBatchInstrumentationError( + "skipped context injection", + "invalid metadata", + ); + return undefined; + } + const metadata = metadataValue ?? {}; + if ( + Object.prototype.hasOwnProperty.call( + metadata, + BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY, + ) + ) { + logBatchInstrumentationError( + "skipped context injection", + "conflicting reserved metadata", + ); + return undefined; + } + if (Object.keys(metadata).length >= 16) { + logBatchInstrumentationError( + "skipped context injection", + "metadata is full", + ); + return undefined; + } + + const exportedParent = await exportMinimalParent(parent); + if (!exportedParent) { + logBatchInstrumentationError( + "skipped context injection", + "no routable Braintrust parent", + ); + return undefined; + } + const signingSecret = + _internalGetGlobalState()._internalGetTraceContextSigningSecret(); + if (!signingSecret) { + logBatchInstrumentationError( + "skipped context injection", + "no signing secret is available", + ); + return undefined; + } + const inputData = await readBatchInputs(args.input, endpoint); + if (inputData.issues.length > 0) { + logBatchInstrumentationError( + "skipped invalid input file", + inputData.issues[0], + ); + return undefined; + } + const context: BatchContext = { + version: 1, + parent: exportedParent, + inputFileId, + inputDigest: inputData.inputDigest, + endpoint, + startTime, + operationNonce: randomBatchNonce(), + }; + const signature = await signBatchContext(context, signingSecret); + if (!signature) { + logBatchInstrumentationError( + "skipped context injection", + "could not sign batch context", + ); + return undefined; + } + const serialized = JSON.stringify({ + version: context.version, + parent: context.parent, + inputDigest: context.inputDigest, + startTime: context.startTime, + operationNonce: context.operationNonce, + signature, + } satisfies SerializedBatchContext); + if (serialized.length > 512) { + logBatchInstrumentationError( + "skipped context injection", + "signed context exceeds OpenAI metadata limits", + ); + return undefined; + } + + return { + context, + inputs: inputData.inputs, + params: { + ...params, + metadata: { + ...metadata, + [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: serialized, + }, + }, + }; +} + +async function deterministicDigest(namespace: string, ...parts: string[]) { + const encoded = new TextEncoder().encode( + [namespace, ...parts].map((part) => `${part.length}:${part}`).join("\0"), + ); + return new Uint8Array( + await globalThis.crypto.subtle.digest("SHA-256", encoded), + ); +} + +function digestHex(bytes: Uint8Array, length: number): string { + return Array.from(bytes.slice(0, length)) + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +function digestUuid(bytes: Uint8Array): string { + const hex = digestHex(bytes, 16); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} + +function randomBatchNonce(): string { + return digestHex(globalThis.crypto.getRandomValues(new Uint8Array(16)), 16); +} + +async function batchSpanIds(operationNonce: string): Promise<{ + rowId: string; + spanId: string; + rootSpanId: string; +}> { + const [row, span, root] = await Promise.all([ + deterministicDigest("openai:batch:row", operationNonce), + deterministicDigest("openai:batch:span", operationNonce), + deterministicDigest("openai:batch:root", operationNonce), + ]); + return { + rowId: digestUuid(row), + spanId: digestHex(span, 8), + rootSpanId: digestHex(root, 16), + }; +} + +async function childSpanIds(operationNonce: string, customId: string) { + const [row, span] = await Promise.all([ + deterministicDigest("openai:batch:child:row", operationNonce, customId), + deterministicDigest("openai:batch:child:span", operationNonce, customId), + ]); + return { rowId: digestUuid(row), spanId: digestHex(span, 8) }; +} + +async function batchContextFromBatch( + batch: OpenAIBatchLike, +): Promise { + const metadata = read(batch, "metadata"); + const inputFileId = read(batch, "input_file_id"); + const endpoint = read(batch, "endpoint"); + if ( + !isObject(metadata) || + typeof inputFileId !== "string" || + typeof endpoint !== "string" + ) { + return undefined; + } + return await parseBatchContext( + read(metadata, BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY), + inputFileId, + endpoint, + ); +} + +async function startBatchSpan(context: BatchContext): Promise { + const ids = await batchSpanIds(context.operationNonce); + const parent = SpanComponentsV4.fromStr(context.parent); + const hasParentSpan = Boolean( + parent.data.row_id && parent.data.span_id && parent.data.root_span_id, + ); + return withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMergeAndParentSpanIds( + withSpanInstrumentationName( + { + name: "openai.batch", + type: SpanTypeAttribute.TASK, + parent: context.parent, + ...(!hasParentSpan + ? { + parentSpanIds: { + parentSpanIds: [], + rootSpanId: ids.rootSpanId, + }, + } + : {}), + spanId: ids.spanId, + startTime: context.startTime, + event: { + id: ids.rowId, + metadata: { provider: "openai" }, + }, + }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ), + ); +} + +async function startBatchChild( + context: BatchContext, + taskParent: string, + input: BatchInputRecord, +): Promise { + const ids = await childSpanIds(context.operationNonce, input.customId); + return withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMerge( + withSpanInstrumentationName( + { + name: + context.endpoint === "/v1/chat/completions" + ? "Chat Completion" + : "openai.responses.create", + type: SpanTypeAttribute.LLM, + parent: taskParent, + spanId: ids.spanId, + startTime: context.startTime, + event: { + id: ids.rowId, + ...(input.spanData?.input !== undefined + ? { input: input.spanData.input } + : {}), + metadata: { + ...input.spanData?.metadata, + provider: "openai", + }, + }, + }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ), + ); +} + +async function* jsonlRecords( + file: OpenAIBatchJSONL | OpenAIBatchResultFile, + onIssue: (error: Error) => void = () => {}, +): AsyncGenerator { + const resolvedFile = await file; + if (resolvedFile === undefined) { + return; + } + if (typeof resolvedFile === "string") { + for (const line of resolvedFile.split(/\r?\n/)) { + if (!line.trim()) { + continue; + } + try { + yield JSON.parse(line); + } catch (error) { + logBatchInstrumentationError("skipped malformed JSONL", error); + onIssue(new Error("OpenAI Batch file contains malformed JSONL")); + } + } + return; + } + + const clone = read(resolvedFile, "clone"); + let response = resolvedFile; + if (typeof clone === "function") { + try { + response = Reflect.apply(clone, resolvedFile, []); + } catch (error) { + logBatchInstrumentationError("could not clone JSONL response", error); + onIssue(new Error("OpenAI Batch response could not be cloned")); + return; + } + } + const body = read(response, "body"); + const getReader = read(body, "getReader"); + if (isObject(body) && typeof getReader === "function") { + let reader: unknown; + try { + reader = Reflect.apply(getReader, body, []); + if (!isObject(reader)) { + throw new Error("Response body returned an invalid stream reader"); + } + + const decoder = new TextDecoder(); + let pending = ""; + while (true) { + const readChunk = read(reader, "read"); + if (typeof readChunk !== "function") { + throw new Error("Response body stream reader has no read method"); + } + const chunk = await Reflect.apply(readChunk, reader, []); + if (!isObject(chunk)) { + throw new Error("Response body stream returned an invalid chunk"); + } + if (chunk.done === true) { + pending += decoder.decode(); + break; + } + if (!(chunk.value instanceof Uint8Array)) { + throw new Error("Response body stream returned a non-byte chunk"); + } + + pending += decoder.decode(chunk.value, { stream: true }); + let newline = pending.indexOf("\n"); + while (newline !== -1) { + const line = pending.slice(0, newline).replace(/\r$/, ""); + pending = pending.slice(newline + 1); + if (line.trim()) { + try { + yield JSON.parse(line); + } catch (error) { + logBatchInstrumentationError("skipped malformed JSONL", error); + onIssue( + new Error("OpenAI Batch response contains malformed JSONL"), + ); + } + } + newline = pending.indexOf("\n"); + } + } + + if (pending.trim()) { + try { + yield JSON.parse(pending.replace(/\r$/, "")); + } catch (error) { + logBatchInstrumentationError("skipped malformed JSONL", error); + onIssue(new Error("OpenAI Batch response contains malformed JSONL")); + } + } + } catch (error) { + logBatchInstrumentationError("could not read JSONL response body", error); + onIssue(new Error("OpenAI Batch response body could not be read")); + } finally { + const releaseLock = read(reader, "releaseLock"); + if (isObject(reader) && typeof releaseLock === "function") { + try { + Reflect.apply(releaseLock, reader, []); + } catch (error) { + logBatchInstrumentationError( + "could not release stream reader", + error, + ); + } + } + } + return; + } + + if (isBatchRecordIterable(resolvedFile)) { + for await (const record of resolvedFile) { + yield record; + } + } else { + logBatchInstrumentationError("skipped invalid JSONL source", resolvedFile); + onIssue(new Error("OpenAI Batch file source is invalid")); + } +} + +async function startPreparedBatch( + inputs: Map, + context: BatchContext, +): Promise { + const task = await startBatchSpan(context); + const taskParent = await task.export(); + for (const input of inputs.values()) { + try { + await startBatchChild(context, taskParent, input); + } catch (error) { + logBatchInstrumentationError("skipped invalid input record", error); + } + } +} + +export const interceptOpenAIBatchCreate: Parameters< + typeof openAIChannels.batchesCreate.intercept +>[0] = async (target, thisArg, args) => { + const parent = getSpanParentObject(); + const startTime = getCurrentUnixTimestamp(); + let prepared: PreparedBatch | undefined; + try { + prepared = await prepareCreateParams(args[0], parent, startTime); + } catch (error) { + logBatchInstrumentationError("could not prepare batch", error); + } + + const batch = await Reflect.apply(target, thisArg, [ + prepared + ? { + ...args[0], + params: prepared.params, + } + : args[0], + ]); + if (prepared) { + try { + const returnedContext = await batchContextFromBatch(batch); + if ( + !returnedContext || + batchContextPayload(returnedContext) !== + batchContextPayload(prepared.context) + ) { + logBatchInstrumentationError( + "did not start batch spans", + "returned batch context does not match the submitted batch", + ); + } else { + await startPreparedBatch(prepared.inputs, prepared.context); + } + } catch (error) { + logBatchInstrumentationError("could not start batch spans", error); + } + } + return batch; +}; + +function errorFromResult(result: BatchResultRecord): Error | undefined { + const error = read(result.value, "error"); + if (isObject(error)) { + const message = read(error, "message"); + return new Error( + typeof message === "string" ? message : "OpenAI Batch request failed", + ); + } + const response = read(result.value, "response"); + const statusCode = read(response, "status_code"); + if ( + result.source === "error" || + (typeof statusCode === "number" && (statusCode < 200 || statusCode >= 300)) + ) { + const body = read(response, "body"); + const bodyError = read(body, "error"); + const message = read(bodyError, "message"); + return new Error( + typeof message === "string" ? message : "OpenAI Batch request failed", + ); + } + return undefined; +} + +async function completeBatchResult( + context: BatchContext, + taskParent: string, + endTime: number, + input: BatchInputRecord, + result: BatchResultRecord, +): Promise { + const child = await startBatchChild(context, taskParent, input); + try { + const resultError = errorFromResult(result); + const response = read(result.value, "response"); + const responseBody = read(response, "body"); + if (resultError) { + child.log({ error: resultError }); + } else if (isObject(responseBody)) { + if (context.endpoint === "/v1/chat/completions") { + const model = read(responseBody, "model"); + child.log({ + output: read(responseBody, "choices"), + ...(typeof model === "string" ? { metadata: { model } } : {}), + metrics: parseMetricsFromUsage(read(responseBody, "usage")), + }); + } else { + const model = read(responseBody, "model"); + child.log({ + output: processImagesInOutput(read(responseBody, "output")), + ...(typeof model === "string" ? { metadata: { model } } : {}), + metrics: parseMetricsFromUsage(read(responseBody, "usage")), + }); + } + } else { + child.log({ error: new Error("OpenAI Batch response body is missing") }); + } + } catch (error) { + child.log({ error }); + } finally { + child.end({ endTime }); + } +} + +async function readBatchInputs( + file: OpenAIBatchResultFile, + endpoint: string, +): Promise<{ + inputDigest: string; + inputs: Map; + issues: Error[]; +}> { + const inputs = new Map(); + const issues: Error[] = []; + const chunkDigests: string[] = []; + let digestChunk = ""; + try { + for await (const value of jsonlRecords(file, (issue) => + issues.push(issue), + )) { + const customId = read(value, "custom_id"); + const body = read(value, "body"); + if ( + !validCustomId(customId) || + inputs.has(customId) || + read(value, "method") !== "POST" || + read(value, "url") !== endpoint || + !isObject(body) + ) { + issues.push(new Error("OpenAI Batch input contains an invalid record")); + continue; + } + const canonicalRecord = JSON.stringify({ + body: canonicalizeJSON(body), + custom_id: customId, + }); + const framedRecord = `${canonicalRecord.length}:${canonicalRecord}`; + if ( + digestChunk.length > 0 && + digestChunk.length + framedRecord.length > INPUT_DIGEST_CHUNK_SIZE + ) { + chunkDigests.push( + digestHex( + await deterministicDigest("openai:batch:input:chunk", digestChunk), + 32, + ), + ); + digestChunk = ""; + } + digestChunk += framedRecord; + + let spanData: BatchInputRecord["spanData"]; + try { + spanData = extractOpenAIBatchInput(endpoint, body); + } catch (error) { + logBatchInstrumentationError("could not extract batch input", error); + } + inputs.set(customId, { customId, spanData }); + } + } catch (error) { + logBatchInstrumentationError("could not process input file", error); + issues.push(new Error("OpenAI Batch input file could not be processed")); + } + chunkDigests.push( + digestHex( + await deterministicDigest("openai:batch:input:chunk", digestChunk), + 32, + ), + ); + return { + inputDigest: digestHex( + await deterministicDigest("openai:batch:input", chunkDigests.join("")), + 32, + ), + inputs, + issues, + }; +} + +function canonicalizeJSON( + value: unknown, + ancestors: Set = new Set(), +): unknown { + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" + ) { + return value; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) { + throw new Error("OpenAI Batch input contains a non-finite number"); + } + return value; + } + if (Array.isArray(value)) { + if (ancestors.has(value)) { + throw new Error("OpenAI Batch input contains a circular value"); + } + ancestors.add(value); + try { + return value.map((entry) => canonicalizeJSON(entry, ancestors)); + } finally { + ancestors.delete(value); + } + } + if (!isObject(value)) { + throw new Error("OpenAI Batch input contains a non-JSON value"); + } + if (ancestors.has(value)) { + throw new Error("OpenAI Batch input contains a circular value"); + } + ancestors.add(value); + try { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => { + const entry = read(value, key); + if (entry === undefined) { + throw new Error("OpenAI Batch input contains an undefined value"); + } + return [key, canonicalizeJSON(entry, ancestors)]; + }), + ); + } finally { + ancestors.delete(value); + } +} + +async function completeResultFile({ + context, + endTime, + file, + inputs, + issues, + seen, + source, + taskParent, +}: { + context: BatchContext; + endTime: number; + file: OpenAIBatchResultFile; + inputs: Map; + issues: Error[]; + seen: Set; + source: BatchResultRecord["source"]; + taskParent: string; +}): Promise { + try { + for await (const value of jsonlRecords(file, (issue) => + issues.push(issue), + )) { + try { + const customId = read(value, "custom_id"); + if (!validCustomId(customId) || !isObject(value)) { + logBatchInstrumentationError( + "skipped result without custom_id", + value, + ); + issues.push( + new Error("OpenAI Batch result is missing a valid custom_id"), + ); + continue; + } + if (seen.has(customId)) { + logBatchInstrumentationError("skipped duplicate result", customId); + issues.push(new Error("OpenAI Batch result contains a duplicate")); + continue; + } + const input = inputs.get(customId); + if (!input) { + issues.push( + new Error("OpenAI Batch result does not match an input record"), + ); + continue; + } + seen.add(customId); + await completeBatchResult(context, taskParent, endTime, input, { + value, + source, + }); + } catch (error) { + logBatchInstrumentationError("skipped invalid result", error); + issues.push(new Error("OpenAI Batch result could not be processed")); + } + } + } catch (error) { + logBatchInstrumentationError(`could not process ${source} file`, error); + issues.push( + new Error(`OpenAI Batch ${source} file could not be processed`), + ); + } +} + +function terminalEndTime( + batch: OpenAIBatchLike, + webhookCreatedAt: number | undefined, + startTime: number, +): number { + const timestamp = + batch.status === "completed" + ? batch.completed_at + : batch.status === "failed" + ? batch.failed_at + : batch.status === "expired" + ? batch.expired_at + : batch.cancelled_at; + return typeof timestamp === "number" && + Number.isFinite(timestamp) && + timestamp >= startTime + ? timestamp + : typeof webhookCreatedAt === "number" && + Number.isFinite(webhookCreatedAt) && + webhookCreatedAt >= startTime + ? webhookCreatedAt + : Math.max(getCurrentUnixTimestamp(), startTime); +} + +async function completeBatch( + args: CompleteOpenAIBatchArgs, +): Promise { + const { batch, webhook } = args; + const batchId = read(batch, "id"); + const status = read(batch, "status"); + if ( + typeof batchId !== "string" || + typeof status !== "string" || + !TERMINAL_STATUSES.has(status) + ) { + return; + } + if ( + webhook && + (read(webhook, "type") !== `batch.${status}` || + read(read(webhook, "data"), "id") !== batchId) + ) { + logBatchInstrumentationError("ignored mismatched webhook", webhook); + return; + } + const context = await batchContextFromBatch(batch); + if (!context || !SUPPORTED_ENDPOINTS.has(context.endpoint)) { + return; + } + + const endTime = terminalEndTime( + batch, + webhook?.created_at, + context.startTime, + ); + const inputData = await readBatchInputs(args.input, context.endpoint); + const issues = [...inputData.issues]; + if ( + inputData.issues.length === 0 && + inputData.inputDigest !== context.inputDigest + ) { + issues.push( + new Error("OpenAI Batch input does not match the signed input file"), + ); + } + const requestTotal = read(read(batch, "request_counts"), "total"); + if ( + typeof requestTotal === "number" && + requestTotal !== inputData.inputs.size + ) { + issues.push( + new Error("OpenAI Batch input does not match its request count"), + ); + } + if (issues.length > 0) { + logBatchInstrumentationError("left batch spans pending", issues[0]); + return; + } + + const task = await startBatchSpan(context); + const taskParent = await task.export(); + const seen = new Set(); + await Promise.all([ + completeResultFile({ + context, + endTime, + file: args.outputFile, + inputs: inputData.inputs, + issues, + seen, + source: "output", + taskParent, + }), + completeResultFile({ + context, + endTime, + file: args.errorFile, + inputs: inputData.inputs, + issues, + seen, + source: "error", + taskParent, + }), + ]); + if (issues.length > 0) { + logBatchInstrumentationError("left batch spans pending", issues[0]); + return; + } + + const missing = [...inputData.inputs.values()].filter( + ({ customId }) => !seen.has(customId), + ); + if (status === "completed" && missing.length > 0) { + logBatchInstrumentationError( + "left batch spans pending", + new Error("OpenAI Batch result files are incomplete"), + ); + return; + } + for (const input of missing) { + try { + const child = await startBatchChild(context, taskParent, input); + child.log({ error: new Error(`OpenAI Batch ${status}`) }); + child.end({ endTime }); + } catch (error) { + logBatchInstrumentationError("could not complete missing result", error); + } + } + + if (status !== "completed") { + task.log({ error: new Error(`OpenAI Batch ${status}`) }); + } + task.end({ endTime }); +} + +export const interceptOpenAIBatchComplete: Parameters< + typeof openAIChannels.batchesComplete.intercept +>[0] = (target, thisArg, args) => + Promise.resolve(Reflect.apply(target, thisArg, args)).then(async (batch) => { + try { + await completeBatch({ ...args[0], batch }); + } catch (error) { + logBatchInstrumentationError("could not complete batch", error); + } + return batch; + }); diff --git a/js/src/instrumentation/plugins/openai-channels.ts b/js/src/instrumentation/plugins/openai-channels.ts index 4b1347609..1d27b0366 100644 --- a/js/src/instrumentation/plugins/openai-channels.ts +++ b/js/src/instrumentation/plugins/openai-channels.ts @@ -17,6 +17,11 @@ import type { OpenAIResponseCreateParams, OpenAIResponseStreamEvent, } from "../../vendor-sdk-types/openai"; +import type { + CompleteOpenAIBatchArgs, + CreateOpenAIBatchArgs, + OpenAIBatchLike, +} from "../../openai-batch-types"; type OpenAIChatSpanInfo = NonNullable["span_info"]>; @@ -31,6 +36,24 @@ type OpenAIResponsesChannelExtras = OpenAIChannelExtras; export const openAIChannels = defineChannels( "openai", { + batchesCreate: channel< + [CreateOpenAIBatchArgs], + OpenAIBatchLike, + OpenAIChannelExtras + >({ + channelName: "batches.create-resumable", + kind: "async", + }), + + batchesComplete: channel< + [CompleteOpenAIBatchArgs], + OpenAIBatchLike, + OpenAIChannelExtras + >({ + channelName: "batches.complete", + kind: "async", + }), + chatCompletionsCreate: channel< [OpenAIChatCreateParams], OpenAIChatCompletion | OpenAIChatStream, diff --git a/js/src/instrumentation/plugins/openai-plugin.test.ts b/js/src/instrumentation/plugins/openai-plugin.test.ts index 5f2b237fa..b55e56d8a 100644 --- a/js/src/instrumentation/plugins/openai-plugin.test.ts +++ b/js/src/instrumentation/plugins/openai-plugin.test.ts @@ -1,9 +1,9 @@ import { describe, it, expect } from "vitest"; import { parseMetricsFromUsage, - processImagesInOutput, aggregateChatCompletionChunks, } from "./openai-plugin"; +import { processImagesInOutput } from "./openai-span-data"; import { Attachment } from "../../logger"; describe("parseMetricsFromUsage", () => { diff --git a/js/src/instrumentation/plugins/openai-plugin.ts b/js/src/instrumentation/plugins/openai-plugin.ts index d1ca89dcc..0ee68a344 100644 --- a/js/src/instrumentation/plugins/openai-plugin.ts +++ b/js/src/instrumentation/plugins/openai-plugin.ts @@ -5,11 +5,19 @@ import { traceSyncStreamChannel, unsubscribeAll, } from "../core/channel-tracing"; -import { Attachment } from "../../logger"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; -import { processInputAttachments } from "../../wrappers/attachment-utils"; import { openAIChannels } from "./openai-channels"; +import { + extractOpenAIChatInput, + extractOpenAIResponsesInput, + extractOpenAIResponsesMetadata, + processImagesInOutput, +} from "./openai-span-data"; +import { + interceptOpenAIBatchComplete, + interceptOpenAIBatchCreate, +} from "./openai-batch-instrumentation"; import { BRAINTRUST_CACHED_STREAM_METRIC, getCachedMetricFromHeaders, @@ -38,18 +46,17 @@ export class OpenAIPlugin extends BasePlugin { } protected onEnable(): void { + this.unsubscribers.push( + openAIChannels.batchesCreate.intercept(interceptOpenAIBatchCreate), + openAIChannels.batchesComplete.intercept(interceptOpenAIBatchComplete), + ); + // Chat Completions - supports streaming this.unsubscribers.push( traceStreamingChannel(openAIChannels.chatCompletionsCreate, { name: "Chat Completion", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIChatInput(params), extractOutput: (result) => { return result?.choices; }, @@ -101,13 +108,7 @@ export class OpenAIPlugin extends BasePlugin { traceStreamingChannel(openAIChannels.betaChatCompletionsParse, { name: "Chat Completion", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIChatInput(params), extractOutput: (result) => { return result?.choices; }, @@ -131,13 +132,7 @@ export class OpenAIPlugin extends BasePlugin { traceSyncStreamChannel(openAIChannels.betaChatCompletionsStream, { name: "Chat Completion", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIChatInput(params), }), ); @@ -171,23 +166,11 @@ export class OpenAIPlugin extends BasePlugin { traceStreamingChannel(openAIChannels.responsesCreate, { name: "openai.responses.create", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractOutput: (result) => { return processImagesInOutput(result?.output); }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, + extractMetadata: (result) => extractOpenAIResponsesMetadata(result), extractMetrics: (result, startTime, endEvent) => { const metrics = withCachedMetric( parseMetricsFromUsage(result?.usage), @@ -208,13 +191,7 @@ export class OpenAIPlugin extends BasePlugin { traceSyncStreamChannel(openAIChannels.responsesStream, { name: "openai.responses.create", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractFromEvent: (event) => { if (event.type !== "response.completed" || !event.response) { return {}; @@ -243,23 +220,11 @@ export class OpenAIPlugin extends BasePlugin { traceStreamingChannel(openAIChannels.responsesParse, { name: "openai.responses.parse", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractOutput: (result) => { return processImagesInOutput(result?.output); }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, + extractMetadata: (result) => extractOpenAIResponsesMetadata(result), extractMetrics: (result, startTime, endEvent) => { const metrics = withCachedMetric( parseMetricsFromUsage(result?.usage), @@ -280,23 +245,11 @@ export class OpenAIPlugin extends BasePlugin { traceAsyncChannel(openAIChannels.responsesCompact, { name: "openai.responses.compact", type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, + extractInput: ([params]) => extractOpenAIResponsesInput(params), extractOutput: (result) => { return processImagesInOutput(result?.output); }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, + extractMetadata: (result) => extractOpenAIResponsesMetadata(result), extractMetrics: (result, startTime, endEvent) => { const metrics = withCachedMetric( parseMetricsFromUsage(result?.usage), @@ -370,54 +323,6 @@ function withCachedMetric( }; } -/** - * Process output to convert base64 images to attachments. - * Used for Responses API image generation output. - */ -export function processImagesInOutput(output: any): any { - if (Array.isArray(output)) { - return output.map(processImagesInOutput); - } - - if (isObject(output)) { - if ( - output.type === "image_generation_call" && - output.result && - typeof output.result === "string" - ) { - const fileExtension = output.output_format || "png"; - const contentType = `image/${fileExtension}`; - - const baseFilename = - output.revised_prompt && typeof output.revised_prompt === "string" - ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") - : "generated_image"; - const filename = `${baseFilename}.${fileExtension}`; - - // Convert base64 string to Blob - const binaryString = atob(output.result); - const bytes = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - const blob = new Blob([bytes], { type: contentType }); - - const attachment = new Attachment({ - data: blob, - filename: filename, - contentType: contentType, - }); - - return { - ...output, - result: attachment, - }; - } - } - - return output; -} - function mergeLogprobTokens( existing: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], incoming: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], diff --git a/js/src/instrumentation/plugins/openai-span-data.ts b/js/src/instrumentation/plugins/openai-span-data.ts new file mode 100644 index 000000000..2562315ad --- /dev/null +++ b/js/src/instrumentation/plugins/openai-span-data.ts @@ -0,0 +1,120 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { Attachment } from "../../logger"; +import { processInputAttachments } from "../../wrappers/attachment-utils"; +import { isObject } from "../../../util/index"; + +const OPENAI_METADATA_KEYS = [ + "model", + "temperature", + "top_p", + "max_tokens", + "frequency_penalty", + "presence_penalty", + "stop", + "response_format", + "tools", + "tool_choice", + "parallel_tool_calls", + "max_tool_calls", +] as const; + +function batchMetadata(params: Record) { + const metadata: Record = { provider: "openai" }; + for (const key of OPENAI_METADATA_KEYS) { + try { + const value = Reflect.get(params, key); + if (value !== undefined) { + metadata[key] = value; + } + } catch { + // Ignore hostile getters in provider input. + } + } + return metadata; +} + +export function extractOpenAIBatchInput( + endpoint: string, + params: Record, +): { input: unknown; metadata: Record } { + const input = + endpoint === "/v1/chat/completions" ? params.messages : params.input; + return { + input: processInputAttachments(input), + metadata: batchMetadata(params), + }; +} + +export function extractOpenAIChatInput(params: Record): { + input: unknown; + metadata: Record; +} { + const { messages, ...metadata } = params; + return { + input: processInputAttachments(messages), + metadata: { ...metadata, provider: "openai" }, + }; +} + +export function extractOpenAIResponsesInput(params: Record): { + input: unknown; + metadata: Record; +} { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; +} + +export function extractOpenAIResponsesMetadata( + result: Record | undefined, +): Record | undefined { + if (!result) { + return undefined; + } + const { output: _output, usage: _usage, ...metadata } = result; + return Object.keys(metadata).length > 0 ? metadata : undefined; +} + +/** Convert Responses API base64 image outputs to Braintrust attachments. */ +export function processImagesInOutput(output: any): any { + if (Array.isArray(output)) { + return output.map(processImagesInOutput); + } + + if ( + isObject(output) && + output.type === "image_generation_call" && + typeof output.result === "string" && + output.result + ) { + const fileExtension = output.output_format || "png"; + const contentType = `image/${fileExtension}`; + const baseFilename = + typeof output.revised_prompt === "string" + ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") + : "generated_image"; + let binaryString: string; + try { + binaryString = atob(output.result); + } catch { + return output; + } + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + return { + ...output, + result: new Attachment({ + data: new Blob([bytes], { type: contentType }), + filename: `${baseFilename}.${fileExtension}`, + contentType, + }), + }; + } + + return output; +} diff --git a/js/src/logger.ts b/js/src/logger.ts index 9733c3030..4c545a833 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -6363,6 +6363,22 @@ export function _internalStartSpanWithInitialMerge< InitialSpanWriteAsMergeArg).span; } +/** @internal Start a deterministic span under an exported object-only parent. */ +export function _internalStartSpanWithInitialMergeAndParentSpanIds< + IsAsyncFlush extends boolean = true, +>(args: StartSpanArgs & AsyncFlushArg & OptionalStateArg): Span { + return startSpanAndIsLogger( + { + ...args, + [INITIAL_SPAN_WRITE_AS_MERGE]: true, + } as StartSpanArgs & + AsyncFlushArg & + OptionalStateArg & + InitialSpanWriteAsMergeArg, + { useParentSpanIdsForObjectParent: true }, + ).span; +} + /** @internal Start a span with SDK-controlled context fields. */ export function _internalStartSpanWithContext< IsAsyncFlush extends boolean = true, @@ -6399,6 +6415,7 @@ function startSpanAndIsLogger( AsyncFlushArg & OptionalStateArg & InternalSpanContextArg, + internalOptions?: { useParentSpanIdsForObjectParent?: boolean }, ): { span: Span; isSyncFlushLogger: boolean } { const state = args?.state ?? _globalState; @@ -6416,7 +6433,7 @@ function startSpanAndIsLogger( parentObject instanceof SpanComponentsV3 || parentObject instanceof SpanComponentsV4 ) { - const parentSpanIds: ParentSpanIds | undefined = + const parentSpanIds: ParentSpanIds | MultiParentSpanIds | undefined = parentObject.data.row_id && parentSpanIdsUsable( parentObject.data.span_id, @@ -6426,7 +6443,9 @@ function startSpanAndIsLogger( spanId: parentObject.data.span_id, rootSpanId: parentObject.data.root_span_id, } - : undefined; + : internalOptions?.useParentSpanIdsForObjectParent + ? args?.parentSpanIds + : undefined; // The parent object/state are already resolved from `parent` above; drop // the raw `parent` so it isn't re-normalized. const { parent: _ignoredParent, ...spanArgs } = args ?? {}; diff --git a/js/src/openai-batch-types.ts b/js/src/openai-batch-types.ts new file mode 100644 index 000000000..f21a0540b --- /dev/null +++ b/js/src/openai-batch-types.ts @@ -0,0 +1,82 @@ +export type OpenAIBatchJSONL = + | string + | Iterable + | AsyncIterable; + +type OpenAIBatchResultFileContent = OpenAIBatchJSONL | Response; + +type OpenAIBatchResultFile = + | OpenAIBatchResultFileContent + | PromiseLike; + +type SupportedOpenAIBatchEndpoint = "/v1/chat/completions" | "/v1/responses"; + +export interface OpenAIBatchLike { + id: string; + endpoint: string; + input_file_id: string; + status: string; + completed_at?: number | null; + failed_at?: number | null; + expired_at?: number | null; + cancelled_at?: number | null; + metadata?: Record | null; + request_counts?: { + completed?: number; + failed?: number; + total?: number; + } | null; + [key: string]: unknown; +} + +export interface OpenAIBatchCreateParams { + input_file_id: string; + endpoint: SupportedOpenAIBatchEndpoint; + completion_window: "24h"; + metadata?: Record | null; + [key: string]: unknown; +} + +export interface OpenAIBatchCreateInputParams { + endpoint: SupportedOpenAIBatchEndpoint; + completion_window: "24h"; + metadata?: Record | null; + [key: string]: unknown; +} + +export interface OpenAIFileLike { + id: string; + [key: string]: unknown; +} + +export interface OpenAIBatchClientLike { + batches: { + create(params: OpenAIBatchCreateParams): PromiseLike; + }; +} + +export interface OpenAIBatchWebhookEvent { + type: + | "batch.completed" + | "batch.failed" + | "batch.expired" + | "batch.cancelled"; + created_at?: number; + data: { id: string }; + [key: string]: unknown; +} + +export interface CreateOpenAIBatchArgs { + client: OpenAIBatchClientLike; + inputFile: OpenAIFileLike; + input: OpenAIBatchJSONL; + params: OpenAIBatchCreateInputParams; +} + +export interface CompleteOpenAIBatchArgs { + batch: TBatch; + input: OpenAIBatchResultFile; + outputFile?: OpenAIBatchResultFile; + errorFile?: OpenAIBatchResultFile; + webhook?: OpenAIBatchWebhookEvent; +} diff --git a/js/src/openai-batch.ts b/js/src/openai-batch.ts new file mode 100644 index 000000000..510997816 --- /dev/null +++ b/js/src/openai-batch.ts @@ -0,0 +1,65 @@ +import { openAIChannels } from "./instrumentation/plugins/openai-channels"; +import type { + CompleteOpenAIBatchArgs, + CreateOpenAIBatchArgs, + OpenAIBatchLike, +} from "./openai-batch-types"; + +/** + * Create an OpenAI Batch and start pending Braintrust spans for its requests. + * + * Supported endpoints are `/v1/chat/completions` and `/v1/responses`. + */ +export async function createOpenAIBatch( + args: CreateOpenAIBatchArgs, +): Promise { + const batch = await openAIChannels.batchesCreate.invoke( + async (input) => + await input.client.batches.create({ + ...input.params, + input_file_id: input.inputFile.id, + }), + undefined, + [args], + {}, + ); + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return batch as TBatch; +} + +/** + * Complete pending OpenAI Batch spans from caller-supplied Batch API data. + * + * This helper performs no OpenAI API requests. Pass the Responses or promises + * returned by `client.files.content()` for the input and result files, JSONL + * strings, or iterables of parsed records. + */ +export async function completeOpenAIBatch( + args: CompleteOpenAIBatchArgs, +): Promise { + const input = Promise.resolve(args.input); + const outputFile = + args.outputFile === undefined + ? undefined + : Promise.resolve(args.outputFile); + const errorFile = + args.errorFile === undefined ? undefined : Promise.resolve(args.errorFile); + + // Content requests are commonly started before this helper is called. Attach + // rejection handlers immediately so early no-op paths and disabled + // instrumentation cannot leave caller-supplied promises unhandled. + void input.catch(() => undefined); + void outputFile?.catch(() => undefined); + void errorFile?.catch(() => undefined); + + const batch = await openAIChannels.batchesComplete.invoke( + async (input) => input.batch, + undefined, + [{ ...args, input, outputFile, errorFile }], + {}, + ); + // The completion channel preserves the exact batch object supplied by the + // caller, including fields from a newer OpenAI SDK version. + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return batch as TBatch; +} diff --git a/js/src/openai-promise-utils.test.ts b/js/src/openai-promise-utils.test.ts new file mode 100644 index 000000000..dc474c316 --- /dev/null +++ b/js/src/openai-promise-utils.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from "vitest"; +import { openAIChannels } from "./instrumentation/plugins/openai-channels"; +import { wrapOpenAI } from "./wrappers/oai"; +import { + type APIPromise, + createLazyAPIPromise, +} from "./wrappers/openai-promise-utils"; + +function apiPromise(data: T, requestId: string): APIPromise { + const response = new Response(JSON.stringify(data), { + headers: { "content-type": "application/json" }, + }); + const promise = Promise.resolve(data) as APIPromise; + promise.withResponse = async () => ({ + data, + response, + request_id: requestId, + }); + promise.asResponse = async () => response; + return promise; +} + +describe("OpenAI API promise wrappers", () => { + it("forwards asResponse without parsing the response body", async () => { + const response = new Response(JSON.stringify({ value: "raw" }), { + headers: { "content-type": "application/json" }, + }); + const nativePromise = apiPromise({ value: "parsed" }, "req_raw"); + let tracedWithResponseCalls = 0; + let tracedAsResponseCalls = 0; + const wrapped = createLazyAPIPromise( + async () => { + tracedWithResponseCalls++; + return { data: { value: "parsed" }, response }; + }, + async () => { + tracedAsResponseCalls++; + return response; + }, + () => nativePromise, + ); + + const raw = await wrapped.asResponse(); + expect(tracedWithResponseCalls).toBe(0); + expect(tracedAsResponseCalls).toBe(1); + await expect(raw.json()).resolves.toEqual({ value: "raw" }); + }); + + it("does not trace the request again when parsed after asResponse", async () => { + const response = new Response(JSON.stringify({ value: "raw" }), { + headers: { "content-type": "application/json" }, + }); + const nativePromise = apiPromise({ value: "parsed" }, "req_raw"); + let tracedWithResponseCalls = 0; + let tracedAsResponseCalls = 0; + const wrapped = createLazyAPIPromise( + async () => { + tracedWithResponseCalls++; + return { data: { value: "parsed" }, response }; + }, + async () => { + tracedAsResponseCalls++; + return response; + }, + () => nativePromise, + ); + + await wrapped.asResponse(); + await expect(wrapped).resolves.toEqual({ value: "parsed" }); + expect(tracedWithResponseCalls).toBe(0); + expect(tracedAsResponseCalls).toBe(1); + }); + + it("dispatches the OpenAI channel lifecycle for asResponse-only calls", async () => { + const phases: string[] = []; + let result: unknown; + const tracingChannel = + openAIChannels.chatCompletionsCreate.tracingChannel(); + const handlers = { + asyncEnd: (event: { result?: unknown }) => { + phases.push("asyncEnd"); + result = event.result; + }, + error: () => phases.push("error"), + start: () => phases.push("start"), + }; + tracingChannel.subscribe(handlers); + + try { + const client = wrapOpenAI({ + chat: { + completions: { + create: (_params: { messages: unknown }) => + apiPromise({ choices: [] }, "req_raw"), + }, + }, + embeddings: { create: () => apiPromise({ data: [] }, "req_embed") }, + moderations: { + create: () => apiPromise({ results: [] }, "req_moderation"), + }, + }); + + const response = await client.chat.completions + .create({ messages: [] }) + .asResponse(); + expect(response.bodyUsed).toBe(false); + await new Promise((resolve) => setTimeout(resolve, 0)); + } finally { + tracingChannel.unsubscribe(handlers); + } + + expect(phases).toEqual(["start", "asyncEnd"]); + expect(result).toBeUndefined(); + }); + + it("does not consume a raw response body and preserves cancellation", async () => { + let cancelled = false; + let pulls = 0; + const response = new Response( + new ReadableStream( + { + pull(controller) { + pulls++; + controller.enqueue( + new TextEncoder().encode(JSON.stringify({ choices: [] })), + ); + }, + cancel() { + cancelled = true; + }, + }, + { highWaterMark: 0 }, + ), + { headers: { "content-type": "application/json" } }, + ); + const promise: APIPromise<{ choices: unknown[] }> = Object.assign( + Promise.resolve({ choices: [] as unknown[] }), + { + asResponse: async () => response, + withResponse: async () => ({ data: { choices: [] }, response }), + }, + ); + const phases: string[] = []; + const tracingChannel = + openAIChannels.chatCompletionsCreate.tracingChannel(); + const handlers = { + asyncEnd: () => phases.push("asyncEnd"), + error: () => phases.push("error"), + start: () => phases.push("start"), + }; + tracingChannel.subscribe(handlers); + + try { + const client = wrapOpenAI({ + chat: { + completions: { + create: (_params: { messages: unknown }) => promise, + }, + }, + embeddings: { create: () => apiPromise({ data: [] }, "req_embed") }, + moderations: { + create: () => apiPromise({ results: [] }, "req_moderation"), + }, + }); + + const raw = await client.chat.completions + .create({ messages: [] }) + .asResponse(); + expect(raw).toBe(response); + expect(raw.bodyUsed).toBe(false); + expect(pulls).toBe(0); + expect(phases).toEqual(["start", "asyncEnd"]); + + const reader = raw.body?.getReader(); + await reader?.read(); + await reader?.cancel(); + expect(cancelled).toBe(true); + } finally { + tracingChannel.unsubscribe(handlers); + } + }); + + it.each([false, true])( + "preserves request_id for chat responses with stream=%s", + async (stream) => { + const client = wrapOpenAI({ + chat: { + completions: { + create: (_params: { messages: unknown; stream?: boolean }) => + apiPromise({ choices: [] }, "req_chat"), + }, + }, + embeddings: { create: () => apiPromise({ data: [] }, "req_embed") }, + moderations: { + create: () => apiPromise({ results: [] }, "req_moderation"), + }, + }); + + await expect( + client.chat.completions.create({ messages: [], stream }).withResponse(), + ).resolves.toMatchObject({ request_id: "req_chat" }); + }, + ); +}); diff --git a/js/src/vendor-sdk-types/openai-common.ts b/js/src/vendor-sdk-types/openai-common.ts index d2dca2323..15f88a3c9 100644 --- a/js/src/vendor-sdk-types/openai-common.ts +++ b/js/src/vendor-sdk-types/openai-common.ts @@ -8,11 +8,13 @@ export interface OpenAIAPIPromise extends Promise { withResponse(): Promise>; + asResponse(): Promise; } export interface OpenAIWithResponse { data: T; response: Response; + request_id?: string | null; } // Requests diff --git a/js/src/wrappers/oai.ts b/js/src/wrappers/oai.ts index b2fa0850e..1478b0569 100644 --- a/js/src/wrappers/oai.ts +++ b/js/src/wrappers/oai.ts @@ -27,6 +27,7 @@ import { createLazyAPIPromise, EnhancedResponse, splitSpanInfo, + tracePromiseAsResponse, tracePromiseWithResponse, } from "./openai-promise-utils"; import { OpenAIV4Client } from "../vendor-sdk-types/openai-v4"; @@ -113,7 +114,6 @@ export function wrapOpenAIv4(openai: T): T { OpenAIModerationCreateParams, OpenAIModerationResponse >(typedOpenai.moderations, wrapModerations); - let betaProxy: OpenAIClient["beta"]; if (typedOpenai.beta?.chat?.completions?.stream) { const betaChatCompletionProxy = new Proxy( @@ -148,7 +148,6 @@ export function wrapOpenAIv4(openai: T): T { }); } - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const topLevelProxy = new Proxy(typedOpenai, { get(target, name) { switch (name) { @@ -254,6 +253,12 @@ function wrapChatCompletion< // to avoid unhandled rejections when the underlying OpenAI call fails immediately. // Without lazy execution, the promise chain starts before error handlers are attached. let executionPromise: Promise> | null = null; + let apiPromise: APIPromise | null = null; + + const getAPIPromise = (): APIPromise => { + apiPromise ??= completion(params, options); + return apiPromise; + }; const ensureExecuted = (): Promise> => { if (!executionPromise) { @@ -265,37 +270,48 @@ function wrapChatCompletion< ); if (params.stream) { - const completionPromise = completion( - params, - options, - ) as APIPromise; - const { data, response } = await tracePromiseWithResponse( - openAIChannels.chatCompletionsCreate, - traceContext, - completionPromise, - ); + const completionPromise = + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + getAPIPromise() as APIPromise; + const { data, response, request_id } = + await tracePromiseWithResponse( + openAIChannels.chatCompletionsCreate, + traceContext, + completionPromise, + ); // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return { data: data as C, response }; + return { data: data as C, response, request_id }; } - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const completionResponse = completion( - params, - options, - ) as APIPromise; - const { data, response } = await tracePromiseWithResponse( + const completionResponse = + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + getAPIPromise() as APIPromise; + const { data, response, request_id } = await tracePromiseWithResponse( openAIChannels.chatCompletionsCreate, traceContext, completionResponse, ); // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return { data: data as C, response }; + return { data: data as C, response, request_id }; })(); } return executionPromise; }; - return createLazyAPIPromise(ensureExecuted); + return createLazyAPIPromise( + ensureExecuted, + () => + tracePromiseAsResponse( + openAIChannels.chatCompletionsCreate, + createChannelContext( + openAIChannels.chatCompletionsCreate, + params, + span_info, + ), + getAPIPromise(), + ), + getAPIPromise, + ); }; } @@ -338,6 +354,11 @@ function wrapApiCreateWithChannel< >(allParams); let executionPromise: Promise>> | null = null; + let apiPromise: APIPromise> | null = null; + const getAPIPromise = () => { + apiPromise ??= create(params, options); + return apiPromise; + }; const ensureExecuted = () => { if (!executionPromise) { executionPromise = (async () => { @@ -345,13 +366,22 @@ function wrapApiCreateWithChannel< return tracePromiseWithResponse( channel, traceContext, - create(params, options), + getAPIPromise(), ); })(); } return executionPromise; }; - return createLazyAPIPromise(ensureExecuted); + return createLazyAPIPromise( + ensureExecuted, + () => + tracePromiseAsResponse( + channel, + createChannelContext(channel, params, span_info), + getAPIPromise(), + ), + getAPIPromise, + ); }; } diff --git a/js/src/wrappers/oai_responses.ts b/js/src/wrappers/oai_responses.ts index 8e936ebbc..466856286 100644 --- a/js/src/wrappers/oai_responses.ts +++ b/js/src/wrappers/oai_responses.ts @@ -11,6 +11,7 @@ import { createLazyAPIPromise, EnhancedResponse, splitSpanInfo, + tracePromiseAsResponse, tracePromiseWithResponse, } from "./openai-promise-utils"; @@ -78,6 +79,12 @@ function wrapResponsesAsync< let executionPromise: Promise>> | null = null; + let apiPromise: APIPromise> | null = null; + + const getAPIPromise = () => { + apiPromise ??= target(params, options); + return apiPromise; + }; const ensureExecuted = (): Promise< EnhancedResponse> @@ -85,15 +92,27 @@ function wrapResponsesAsync< if (!executionPromise) { executionPromise = (async () => { const traceContext = createChannelContext(channel, params, span_info); - const apiPromise = target(params, options); - return tracePromiseWithResponse(channel, traceContext, apiPromise); + return tracePromiseWithResponse( + channel, + traceContext, + getAPIPromise(), + ); })(); } return executionPromise; }; - return createLazyAPIPromise(ensureExecuted); + return createLazyAPIPromise( + ensureExecuted, + () => + tracePromiseAsResponse( + channel, + createChannelContext(channel, params, span_info), + getAPIPromise(), + ), + getAPIPromise, + ); }; } diff --git a/js/src/wrappers/openai-promise-utils.ts b/js/src/wrappers/openai-promise-utils.ts index afa23586c..9ba00d0bd 100644 --- a/js/src/wrappers/openai-promise-utils.ts +++ b/js/src/wrappers/openai-promise-utils.ts @@ -11,10 +11,12 @@ import type { export type EnhancedResponse = { response: Response; data: T; + request_id?: string | null; }; export interface APIPromise extends Promise { withResponse(): Promise>; + asResponse(): Promise; } type ChannelContext = @@ -71,19 +73,78 @@ export async function tracePromiseWithResponse< throw new Error("Expected withResponse() to provide response"); } - return { data, response: enhancedResponse.response }; + return { + data, + response: enhancedResponse.response, + request_id: enhancedResponse.request_id, + }; +} + +export async function tracePromiseAsResponse< + TChannel extends OpenAIAsyncChannel, + TResult extends ResultOf, +>( + channel: TChannel, + traceContext: ChannelContext, + apiPromise: APIPromise, +): Promise { + const tracePromise = + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + channel.tracePromise as unknown as ( + fn: () => Promise, + context: ChannelContext, + ) => Promise; + + let response: Response | undefined; + await tracePromise(async () => { + response = await apiPromise.asResponse(); + traceContext.response = response; + return undefined; + }, traceContext); + + if (!response) { + throw new Error("Expected asResponse() to provide response"); + } + return response; } export function createLazyAPIPromise( ensureExecuted: () => Promise>, + ensureResponse: () => Promise, + getAPIPromise: () => APIPromise, ): APIPromise { - let dataPromise: Promise | null = null; + let firstConsumption: "data" | "response" | undefined; + let enhancedResponsePromise: Promise> | undefined; + let dataPromise: Promise | undefined; + let responsePromise: Promise | undefined; + + const withResponse = () => { + firstConsumption ??= "data"; + enhancedResponsePromise ??= + firstConsumption === "data" + ? ensureExecuted() + : getAPIPromise().withResponse(); + return enhancedResponsePromise; + }; + + const asResponse = () => { + firstConsumption ??= "response"; + responsePromise ??= + firstConsumption === "response" + ? ensureResponse() + : getAPIPromise().asResponse(); + return responsePromise; + }; // eslint-disable-next-line @typescript-eslint/consistent-type-assertions return new Proxy({} as APIPromise, { get(target, prop, receiver) { if (prop === "withResponse") { - return () => ensureExecuted(); + return withResponse; + } + + if (prop === "asResponse") { + return asResponse; } if ( @@ -92,9 +153,7 @@ export function createLazyAPIPromise( prop === "finally" || prop in Promise.prototype ) { - if (!dataPromise) { - dataPromise = ensureExecuted().then((result) => result.data); - } + dataPromise ??= withResponse().then((result) => result.data); const value = Reflect.get(dataPromise, prop, receiver); return typeof value === "function" ? value.bind(dataPromise) : value; } From 2d64e44231509df1369f3cef4b9aa660a4921047 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:08:46 +0000 Subject: [PATCH 3/6] Update PR #2369 --- .../openai-batch-api-instrumentation.md | 5 ++ .../openai-v4-auto-hook.span-tree.json | 16 +++--- .../openai-v4-auto-hook.span-tree.txt | 16 +++--- .../openai-v4-latest-auto-hook.span-tree.json | 16 +++--- .../openai-v4-latest-auto-hook.span-tree.txt | 16 +++--- .../openai-v4-latest-wrapped.span-tree.json | 20 +++---- .../openai-v4-latest-wrapped.span-tree.txt | 20 +++---- .../openai-v4-wrapped.span-tree.json | 20 +++---- .../openai-v4-wrapped.span-tree.txt | 20 +++---- .../openai-v5-auto-hook.span-tree.json | 20 +++---- .../openai-v5-auto-hook.span-tree.txt | 20 +++---- .../openai-v5-latest-auto-hook.span-tree.json | 20 +++---- .../openai-v5-latest-auto-hook.span-tree.txt | 20 +++---- .../openai-v5-latest-wrapped.span-tree.json | 24 ++++---- .../openai-v5-latest-wrapped.span-tree.txt | 24 ++++---- .../openai-v5-wrapped.span-tree.json | 24 ++++---- .../openai-v5-wrapped.span-tree.txt | 24 ++++---- .../openai-v6-auto-hook.span-tree.json | 34 ++++++------ .../openai-v6-auto-hook.span-tree.txt | 34 ++++++------ .../openai-v6-latest-auto-hook.span-tree.json | 16 +++--- .../openai-v6-latest-auto-hook.span-tree.txt | 16 +++--- .../openai-v6-latest-wrapped.span-tree.json | 20 +++---- .../openai-v6-latest-wrapped.span-tree.txt | 20 +++---- .../openai-v6-wrapped.span-tree.json | 38 ++++++------- .../openai-v6-wrapped.span-tree.txt | 38 ++++++------- .../openai-instrumentation/scenario.impl.mjs | 6 +- js/src/exports.ts | 5 +- .../openai-batch-instrumentation.test.ts | 55 ++++++++++--------- .../plugins/openai-batch-instrumentation.ts | 18 +++--- .../plugins/openai-channels.ts | 16 +++--- .../instrumentation/plugins/openai-plugin.ts | 12 ++-- js/src/openai-batch-types.ts | 4 +- js/src/openai-batch.ts | 20 +++---- 33 files changed, 346 insertions(+), 331 deletions(-) create mode 100644 .changeset/openai-batch-api-instrumentation.md diff --git a/.changeset/openai-batch-api-instrumentation.md b/.changeset/openai-batch-api-instrumentation.md new file mode 100644 index 000000000..fb7122b69 --- /dev/null +++ b/.changeset/openai-batch-api-instrumentation.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Add OpenAI Batch API Instrumentation diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json index 7d6ea7f6e..1fc78fb5f 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json @@ -899,7 +899,7 @@ { "annotations": [], "logprobs": [], - "text": "PARIS", + "text": "PARIS.", "type": "output_text" } ], @@ -927,7 +927,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "PARIS", + "output_text": "PARIS.", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -976,12 +976,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 3, + "completion_tokens": 4, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 16 + "tokens": 17 } } ], @@ -1438,7 +1438,7 @@ { "annotations": [], "logprobs": [], - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", "type": "output_text" } ], @@ -1466,7 +1466,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1534,12 +1534,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 27, + "completion_tokens": 16, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 73 + "tokens": 62 } } ], diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt index dcb056028..243c2ed0e 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt @@ -800,7 +800,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARIS", + │ "text": "PARIS.", │ "type": "output_text" │ } │ ], @@ -828,7 +828,7 @@ span_tree: │ "model": "gpt-4o-mini-2024-07-18", │ "moderation": null, │ "object": "response", - │ "output_text": "PARIS", + │ "output_text": "PARIS.", │ "parallel_tool_calls": true, │ "presence_penalty": 0, │ "previous_response_id": null, @@ -877,12 +877,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, + │ "completion_tokens": 4, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 16 + │ "tokens": 17 │ } ├── openai-responses-with-response-operation │ metadata: { @@ -1299,7 +1299,7 @@ span_tree: { "annotations": [], "logprobs": [], - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", "type": "output_text" } ], @@ -1327,7 +1327,7 @@ span_tree: "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1395,10 +1395,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 27, + "completion_tokens": 16, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 73 + "tokens": 62 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json index ba468e90c..362408cb3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json @@ -899,7 +899,7 @@ { "annotations": [], "logprobs": [], - "text": "PARIS.", + "text": "PARIS", "type": "output_text" } ], @@ -927,7 +927,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "PARIS.", + "output_text": "PARIS", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -976,12 +976,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 3, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 } } ], @@ -1438,7 +1438,7 @@ { "annotations": [], "logprobs": [], - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", "type": "output_text" } ], @@ -1466,7 +1466,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1534,12 +1534,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 19, + "completion_tokens": 22, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 65 + "tokens": 68 } } ], diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt index de6f2dab6..32d22b074 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt @@ -800,7 +800,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARIS.", + │ "text": "PARIS", │ "type": "output_text" │ } │ ], @@ -828,7 +828,7 @@ span_tree: │ "model": "gpt-4o-mini-2024-07-18", │ "moderation": null, │ "object": "response", - │ "output_text": "PARIS.", + │ "output_text": "PARIS", │ "parallel_tool_calls": true, │ "presence_penalty": 0, │ "previous_response_id": null, @@ -877,12 +877,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 3, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 16 │ } ├── openai-responses-with-response-operation │ metadata: { @@ -1299,7 +1299,7 @@ span_tree: { "annotations": [], "logprobs": [], - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", "type": "output_text" } ], @@ -1327,7 +1327,7 @@ span_tree: "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1395,10 +1395,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 19, + "completion_tokens": 22, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 65 + "tokens": 68 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json index 920f620f5..511c12796 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json @@ -891,7 +891,7 @@ { "annotations": [], "logprobs": [], - "text": "PARIS.", + "text": "PARIS", "type": "output_text" } ], @@ -919,7 +919,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "PARIS.", + "output_text": "PARIS", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -968,12 +968,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 3, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 } } ], @@ -1431,10 +1431,10 @@ "annotations": [], "logprobs": [], "parsed": { - "reasoning": "Adding 20 and 4 gives 24.", + "reasoning": "Adding 20 and 4 gives a total of 24.", "value": 24 }, - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", "type": "output_text" } ], @@ -1463,10 +1463,10 @@ "moderation": null, "object": "response", "output_parsed": { - "reasoning": "Adding 20 and 4 gives 24.", + "reasoning": "Adding 20 and 4 gives a total of 24.", "value": 24 }, - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1534,12 +1534,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 19, + "completion_tokens": 22, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 65 + "tokens": 68 } } ], diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt index c13222807..a8aa0e11c 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt @@ -792,7 +792,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARIS.", + │ "text": "PARIS", │ "type": "output_text" │ } │ ], @@ -820,7 +820,7 @@ span_tree: │ "model": "gpt-4o-mini-2024-07-18", │ "moderation": null, │ "object": "response", - │ "output_text": "PARIS.", + │ "output_text": "PARIS", │ "parallel_tool_calls": true, │ "presence_penalty": 0, │ "previous_response_id": null, @@ -869,12 +869,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 3, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 16 │ } ├── openai-responses-with-response-operation │ metadata: { @@ -1292,10 +1292,10 @@ span_tree: "annotations": [], "logprobs": [], "parsed": { - "reasoning": "Adding 20 and 4 gives 24.", + "reasoning": "Adding 20 and 4 gives a total of 24.", "value": 24 }, - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", "type": "output_text" } ], @@ -1324,10 +1324,10 @@ span_tree: "moderation": null, "object": "response", "output_parsed": { - "reasoning": "Adding 20 and 4 gives 24.", + "reasoning": "Adding 20 and 4 gives a total of 24.", "value": 24 }, - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1395,10 +1395,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 19, + "completion_tokens": 22, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 65 + "tokens": 68 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json index d7968e46f..adf6d5a6b 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json @@ -891,7 +891,7 @@ { "annotations": [], "logprobs": [], - "text": "PARIS", + "text": "PARIS.", "type": "output_text" } ], @@ -919,7 +919,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "PARIS", + "output_text": "PARIS.", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -968,12 +968,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 3, + "completion_tokens": 4, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 16 + "tokens": 17 } } ], @@ -1431,10 +1431,10 @@ "annotations": [], "logprobs": [], "parsed": { - "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "reasoning": "Twenty plus four equals twenty-four.", "value": 24 }, - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", "type": "output_text" } ], @@ -1463,10 +1463,10 @@ "moderation": null, "object": "response", "output_parsed": { - "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "reasoning": "Twenty plus four equals twenty-four.", "value": 24 }, - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1534,12 +1534,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 27, + "completion_tokens": 16, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 73 + "tokens": 62 } } ], diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt index 5c19f6e7b..030c9e215 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt @@ -792,7 +792,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARIS", + │ "text": "PARIS.", │ "type": "output_text" │ } │ ], @@ -820,7 +820,7 @@ span_tree: │ "model": "gpt-4o-mini-2024-07-18", │ "moderation": null, │ "object": "response", - │ "output_text": "PARIS", + │ "output_text": "PARIS.", │ "parallel_tool_calls": true, │ "presence_penalty": 0, │ "previous_response_id": null, @@ -869,12 +869,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, + │ "completion_tokens": 4, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 16 + │ "tokens": 17 │ } ├── openai-responses-with-response-operation │ metadata: { @@ -1292,10 +1292,10 @@ span_tree: "annotations": [], "logprobs": [], "parsed": { - "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "reasoning": "Twenty plus four equals twenty-four.", "value": 24 }, - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", "type": "output_text" } ], @@ -1324,10 +1324,10 @@ span_tree: "moderation": null, "object": "response", "output_parsed": { - "reasoning": "Adding 20 and 4 gives 24, since you combine the two numbers together.", + "reasoning": "Twenty plus four equals twenty-four.", "value": 24 }, - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24, since you combine the two numbers together.\",\"value\":24}", + "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1395,10 +1395,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 27, + "completion_tokens": 16, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 73 + "tokens": 62 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json index b5726d1d3..8d614bc38 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json @@ -1119,7 +1119,7 @@ { "annotations": [], "logprobs": [], - "text": "RESPONSE STREAM", + "text": "RESPONSE STREAM.", "type": "output_text" } ], @@ -1193,12 +1193,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 5, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 18 } } ], @@ -1332,7 +1332,7 @@ { "annotations": [], "logprobs": [], - "text": "PARTIAL.", + "text": "PARTIAL", "type": "output_text" } ], @@ -1405,12 +1405,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 3, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 } } ], @@ -1438,7 +1438,7 @@ { "annotations": [], "logprobs": [], - "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", "type": "output_text" } ], @@ -1466,7 +1466,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1534,12 +1534,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 21, + "completion_tokens": 26, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 67 + "tokens": 72 } } ], diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt index 93c5f56e6..356a57f08 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt @@ -1004,7 +1004,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "RESPONSE STREAM", + │ "text": "RESPONSE STREAM.", │ "type": "output_text" │ } │ ], @@ -1078,12 +1078,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 5, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 18 │ } ├── openai-responses-stream-operation │ metadata: { @@ -1201,7 +1201,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARTIAL.", + │ "text": "PARTIAL", │ "type": "output_text" │ } │ ], @@ -1274,12 +1274,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 3, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 16 │ } └── openai-responses-parse-operation metadata: { @@ -1299,7 +1299,7 @@ span_tree: { "annotations": [], "logprobs": [], - "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", "type": "output_text" } ], @@ -1327,7 +1327,7 @@ span_tree: "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1395,10 +1395,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 21, + "completion_tokens": 26, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 67 + "tokens": 72 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json index e71a94318..d059692a9 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json @@ -1119,7 +1119,7 @@ { "annotations": [], "logprobs": [], - "text": "RESPONSE STREAM.", + "text": "RESPONSE STREAM", "type": "output_text" } ], @@ -1193,12 +1193,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 5, + "completion_tokens": 4, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 18 + "tokens": 17 } } ], @@ -1332,7 +1332,7 @@ { "annotations": [], "logprobs": [], - "text": "PARTIAL.", + "text": "PARTIAL", "type": "output_text" } ], @@ -1405,12 +1405,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 3, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 } } ], @@ -1438,7 +1438,7 @@ { "annotations": [], "logprobs": [], - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", "type": "output_text" } ], @@ -1466,7 +1466,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1534,12 +1534,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 45, + "completion_tokens": 35, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 91 + "tokens": 81 } } ], diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt index 917e99cc2..63909da75 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt @@ -1004,7 +1004,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "RESPONSE STREAM.", + │ "text": "RESPONSE STREAM", │ "type": "output_text" │ } │ ], @@ -1078,12 +1078,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 5, + │ "completion_tokens": 4, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 18 + │ "tokens": 17 │ } ├── openai-responses-stream-operation │ metadata: { @@ -1201,7 +1201,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARTIAL.", + │ "text": "PARTIAL", │ "type": "output_text" │ } │ ], @@ -1274,12 +1274,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 3, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 16 │ } └── openai-responses-parse-operation metadata: { @@ -1299,7 +1299,7 @@ span_tree: { "annotations": [], "logprobs": [], - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", "type": "output_text" } ], @@ -1327,7 +1327,7 @@ span_tree: "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1395,10 +1395,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 45, + "completion_tokens": 35, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 91 + "tokens": 81 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json index 82c330f57..230a290d6 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json @@ -1111,7 +1111,7 @@ { "annotations": [], "logprobs": [], - "text": "RESPONSE STREAM.", + "text": "RESPONSE STREAM", "type": "output_text" } ], @@ -1185,12 +1185,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 5, + "completion_tokens": 4, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 18 + "tokens": 17 } } ], @@ -1324,7 +1324,7 @@ { "annotations": [], "logprobs": [], - "text": "PARTIAL.", + "text": "PARTIAL", "type": "output_text" } ], @@ -1397,12 +1397,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 3, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 } } ], @@ -1431,10 +1431,10 @@ "annotations": [], "logprobs": [], "parsed": { - "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", "value": 24 }, - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", "type": "output_text" } ], @@ -1463,10 +1463,10 @@ "moderation": null, "object": "response", "output_parsed": { - "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", "value": 24 }, - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1534,12 +1534,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 45, + "completion_tokens": 35, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 91 + "tokens": 81 } } ], diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt index 0d5d65a88..1f2c5d78f 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt @@ -996,7 +996,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "RESPONSE STREAM.", + │ "text": "RESPONSE STREAM", │ "type": "output_text" │ } │ ], @@ -1070,12 +1070,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 5, + │ "completion_tokens": 4, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 18 + │ "tokens": 17 │ } ├── openai-responses-stream-operation │ metadata: { @@ -1193,7 +1193,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARTIAL.", + │ "text": "PARTIAL", │ "type": "output_text" │ } │ ], @@ -1266,12 +1266,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 3, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 16 │ } └── openai-responses-parse-operation metadata: { @@ -1292,10 +1292,10 @@ span_tree: "annotations": [], "logprobs": [], "parsed": { - "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", "value": 24 }, - "text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", "type": "output_text" } ], @@ -1324,10 +1324,10 @@ span_tree: "moderation": null, "object": "response", "output_parsed": { - "reasoning": "Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.", + "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", "value": 24 }, - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives 24. When you combine these two numbers, you simply count up from 20 by 4 more units, which results in 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1395,10 +1395,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 45, + "completion_tokens": 35, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 91 + "tokens": 81 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json index a7acdaec3..7deccbd34 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json @@ -1111,7 +1111,7 @@ { "annotations": [], "logprobs": [], - "text": "RESPONSE STREAM", + "text": "RESPONSE STREAM.", "type": "output_text" } ], @@ -1185,12 +1185,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 5, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 18 } } ], @@ -1324,7 +1324,7 @@ { "annotations": [], "logprobs": [], - "text": "PARTIAL.", + "text": "PARTIAL", "type": "output_text" } ], @@ -1397,12 +1397,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 3, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 } } ], @@ -1431,10 +1431,10 @@ "annotations": [], "logprobs": [], "parsed": { - "reasoning": "Adding 20 and 4 together gives us 24.", + "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", "value": 24 }, - "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", "type": "output_text" } ], @@ -1463,10 +1463,10 @@ "moderation": null, "object": "response", "output_parsed": { - "reasoning": "Adding 20 and 4 together gives us 24.", + "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", "value": 24 }, - "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1534,12 +1534,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 21, + "completion_tokens": 26, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 67 + "tokens": 72 } } ], diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt index 9755e3a63..ba711bd95 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt @@ -996,7 +996,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "RESPONSE STREAM", + │ "text": "RESPONSE STREAM.", │ "type": "output_text" │ } │ ], @@ -1070,12 +1070,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 5, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 18 │ } ├── openai-responses-stream-operation │ metadata: { @@ -1193,7 +1193,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARTIAL.", + │ "text": "PARTIAL", │ "type": "output_text" │ } │ ], @@ -1266,12 +1266,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 3, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 16 │ } └── openai-responses-parse-operation metadata: { @@ -1292,10 +1292,10 @@ span_tree: "annotations": [], "logprobs": [], "parsed": { - "reasoning": "Adding 20 and 4 together gives us 24.", + "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", "value": 24 }, - "text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", "type": "output_text" } ], @@ -1324,10 +1324,10 @@ span_tree: "moderation": null, "object": "response", "output_parsed": { - "reasoning": "Adding 20 and 4 together gives us 24.", + "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", "value": 24 }, - "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives us 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1395,10 +1395,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 21, + "completion_tokens": 26, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 67 + "tokens": 72 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json index 5b5261c56..d49b81fe6 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json @@ -1044,7 +1044,7 @@ { "annotations": [], "logprobs": [], - "text": "PARIS.", + "text": "PARIS", "type": "output_text" } ], @@ -1072,7 +1072,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "PARIS.", + "output_text": "PARIS", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1121,12 +1121,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 3, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 } } ], @@ -1264,7 +1264,7 @@ { "annotations": [], "logprobs": [], - "text": "RESPONSE STREAM.", + "text": "RESPONSE STREAM", "type": "output_text" } ], @@ -1338,12 +1338,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 5, + "completion_tokens": 4, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 18 + "tokens": 17 } } ], @@ -1477,7 +1477,7 @@ { "annotations": [], "logprobs": [], - "text": "PARTIAL.", + "text": "PARTIAL", "type": "output_text" } ], @@ -1550,12 +1550,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 3, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 } } ], @@ -1583,7 +1583,7 @@ { "annotations": [], "logprobs": [], - "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", "type": "output_text" } ], @@ -1611,7 +1611,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1679,12 +1679,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 30, + "completion_tokens": 17, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 76 + "tokens": 63 } } ], @@ -1739,7 +1739,7 @@ "type": "message" }, { - "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", "id": "", "type": "compaction" } @@ -1754,12 +1754,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 220, + "completion_tokens": 204, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 133, "time_to_first_token": 0, - "tokens": 353 + "tokens": 337 } } ], diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt index b5864f6a1..b4710287a 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt @@ -929,7 +929,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARIS.", + │ "text": "PARIS", │ "type": "output_text" │ } │ ], @@ -957,7 +957,7 @@ span_tree: │ "model": "gpt-4o-mini-2024-07-18", │ "moderation": null, │ "object": "response", - │ "output_text": "PARIS.", + │ "output_text": "PARIS", │ "parallel_tool_calls": true, │ "presence_penalty": 0, │ "previous_response_id": null, @@ -1006,12 +1006,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 3, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 16 │ } ├── openai-responses-with-response-operation │ metadata: { @@ -1133,7 +1133,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "RESPONSE STREAM.", + │ "text": "RESPONSE STREAM", │ "type": "output_text" │ } │ ], @@ -1207,12 +1207,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 5, + │ "completion_tokens": 4, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 18 + │ "tokens": 17 │ } ├── openai-responses-stream-operation │ metadata: { @@ -1330,7 +1330,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARTIAL.", + │ "text": "PARTIAL", │ "type": "output_text" │ } │ ], @@ -1403,12 +1403,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 3, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 16 │ } ├── openai-responses-parse-operation │ metadata: { @@ -1428,7 +1428,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", │ "type": "output_text" │ } │ ], @@ -1456,7 +1456,7 @@ span_tree: │ "model": "gpt-4o-mini-2024-07-18", │ "moderation": null, │ "object": "response", - │ "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", │ "parallel_tool_calls": true, │ "presence_penalty": 0, │ "previous_response_id": null, @@ -1524,12 +1524,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 30, + │ "completion_tokens": 17, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 46, │ "time_to_first_token": 0, - │ "tokens": 76 + │ "tokens": 63 │ } └── openai-responses-compact-operation metadata: { @@ -1576,7 +1576,7 @@ span_tree: "type": "message" }, { - "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", "id": "", "type": "compaction" } @@ -1591,10 +1591,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 220, + "completion_tokens": 204, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 133, "time_to_first_token": 0, - "tokens": 353 + "tokens": 337 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json index 69ed5df8d..2bbb1383e 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json @@ -1583,7 +1583,7 @@ { "annotations": [], "logprobs": [], - "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", "type": "output_text" } ], @@ -1611,7 +1611,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1679,12 +1679,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 17, + "completion_tokens": 24, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 63 + "tokens": 70 } } ], @@ -1739,7 +1739,7 @@ "type": "message" }, { - "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", "id": "", "type": "compaction" } @@ -1754,12 +1754,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 190, + "completion_tokens": 235, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 133, "time_to_first_token": 0, - "tokens": 323 + "tokens": 368 } } ], @@ -1775,7 +1775,7 @@ } ], "metadata": { - "openaiSdkVersion": "6.48.0", + "openaiSdkVersion": "6.49.0", "scenario": "openai-instrumentation", "testRunId": "" }, diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt index 9e8dca063..254aa8fb6 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt @@ -1,7 +1,7 @@ span_tree: └── openai-instrumentation-root [task] metadata: { - "openaiSdkVersion": "6.48.0", + "openaiSdkVersion": "6.49.0", "scenario": "openai-instrumentation", "testRunId": "" } @@ -1428,7 +1428,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", │ "type": "output_text" │ } │ ], @@ -1456,7 +1456,7 @@ span_tree: │ "model": "gpt-4o-mini-2024-07-18", │ "moderation": null, │ "object": "response", - │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", │ "parallel_tool_calls": true, │ "presence_penalty": 0, │ "previous_response_id": null, @@ -1524,12 +1524,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 17, + │ "completion_tokens": 24, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 46, │ "time_to_first_token": 0, - │ "tokens": 63 + │ "tokens": 70 │ } └── openai-responses-compact-operation metadata: { @@ -1576,7 +1576,7 @@ span_tree: "type": "message" }, { - "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", "id": "", "type": "compaction" } @@ -1591,10 +1591,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 190, + "completion_tokens": 235, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 133, "time_to_first_token": 0, - "tokens": 323 + "tokens": 368 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json index 43b2d172d..2f7daa87b 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json @@ -1576,10 +1576,10 @@ "annotations": [], "logprobs": [], "parsed": { - "reasoning": "20 plus 4 equals 24.", + "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", "value": 24 }, - "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", "type": "output_text" } ], @@ -1608,10 +1608,10 @@ "moderation": null, "object": "response", "output_parsed": { - "reasoning": "20 plus 4 equals 24.", + "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", "value": 24 }, - "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1679,12 +1679,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 17, + "completion_tokens": 24, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 63 + "tokens": 70 } } ], @@ -1739,7 +1739,7 @@ "type": "message" }, { - "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", "id": "", "type": "compaction" } @@ -1754,12 +1754,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 190, + "completion_tokens": 235, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 133, "time_to_first_token": 0, - "tokens": 323 + "tokens": 368 } } ], @@ -1775,7 +1775,7 @@ } ], "metadata": { - "openaiSdkVersion": "6.48.0", + "openaiSdkVersion": "6.49.0", "scenario": "openai-instrumentation", "testRunId": "" }, diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt index 4deb2bed8..b70669bb1 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt @@ -1,7 +1,7 @@ span_tree: └── openai-instrumentation-root [task] metadata: { - "openaiSdkVersion": "6.48.0", + "openaiSdkVersion": "6.49.0", "scenario": "openai-instrumentation", "testRunId": "" } @@ -1421,10 +1421,10 @@ span_tree: │ "annotations": [], │ "logprobs": [], │ "parsed": { - │ "reasoning": "20 plus 4 equals 24.", + │ "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", │ "value": 24 │ }, - │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", │ "type": "output_text" │ } │ ], @@ -1453,10 +1453,10 @@ span_tree: │ "moderation": null, │ "object": "response", │ "output_parsed": { - │ "reasoning": "20 plus 4 equals 24.", + │ "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", │ "value": 24 │ }, - │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", + │ "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", │ "parallel_tool_calls": true, │ "presence_penalty": 0, │ "previous_response_id": null, @@ -1524,12 +1524,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 17, + │ "completion_tokens": 24, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 46, │ "time_to_first_token": 0, - │ "tokens": 63 + │ "tokens": 70 │ } └── openai-responses-compact-operation metadata: { @@ -1576,7 +1576,7 @@ span_tree: "type": "message" }, { - "encrypted_content": "gAAAAABqXybUBbDN8Sjsjp-jAACkdGnXnz_Ik14cbESAS7W-vQOOFviQ9o3W5oTPhY--HPj8mUt4Q5RC911JUKYMVbQFQN2ikRF1tP2qoYfeKyep9i_G1NCkqGYgJrUAEClupevNCW94g5TjH6gOdMfJT2SP4s9_oAvfOo5HB30YpRy_D_UKNqNnSTRlPY6hGjNuf1D-LRHsoWtOl3L_4yUyQmAvDAA-A2eRm6kJETHiPk-Wfhu81zeu29RPnvad1gBNNNpYglWxj_OQZGm4Y1Hd5Kf4MD3I2SxBlWr2bdO2rqzAQJD0NEjK7cKJ40klv8wDiTW7EDwvLoLSKjrYYWxI_05eWfKMRIURscqqfgV22vOrl_cWxctrIwdyiY6Ns_W2ganSvKO8ZXdGxWxwBX0EiDUvbqoMFlSxaJaU7kvxSDy3PIti7_A5VZ1fmTPH7mNoSQaRkDRQftgnMuDdIR2Pjaglm9tFEKJ9NRQe1XtMJvCq0z8mSYzv8ldvwMgnDxPr0umlnzkiJOzblalX8nFNsFM-R6S_dw0n_2E6WqrJNneOVSU51sXqsK9APvoLGJv7Kpaxtdrhhq93_7USMazNM3sxwrrHVY-Et7cYH2lZFT0Zb4gymbNRcel16kLotlX8xE7h6wfLV2UCh11ABtKB-wKVOileaWjAOKPiBCfERNicMYGDL946KzXm9I2akBcd7mFUPRu5Q3JilWbibuwV1ehB5ZFZI3u0LtvHrpVkXQPMUPgV3UQ0JoDgUeAb_WhbXYIDXgER21Uq38xbZ4OOu4Juf5jdY3GTSAK552Bk3s4_Q5jVDRmTQ6DmcOjIK5o_ZJ1ZEOpIEhpH5R4DAwXK-LYFyLG2XFKvyOwMqW0QbNFRCwDk-Wi5PdzrfvQumYGwLQcT2wUksMWUcb32lMrISVit7RyabPRI3vAKhNQ1sM2hyoFnl45tygepVTw75fNAm72_VWvZ0XpETbuhTsmIzHlBnWaZK62oIfY1oVaWyRl_JpPyFPX9S2Gsln0I2AZs_rmt8IyBkiBTGwECmTR1kxbVfpSxStITt7jl8XJim2Pwi325GNnQWt9JMa6xEqTGSloMiaK_dsnremmQkOkmhlROmgcWOqSdGNomf8Aq2NnemWD6Q9fp79uE7joTCHNqKvc6Mf8tqqQFwzJ_Kd0QGFjErKHgU6QAsvOvv-Ybssi7SxWEVhjAg55psNApWLf-qJP0zrGndwqKPMSjwP7WhQMUbLFekqzEsNFDexKPI7N7s1g3SN0fKgm_DSuwQapyZzj7b0pvEYnaby8qvjcHGwHQg-l0cUqBYUPtouHTg8xp2fREc3u_izlgi82QFkHTRoB3KoxwIwbwa-snWmIBuOAJhLB_7LzvUEpq0wFX4bSMS-NRRdo3ZaRdcCcRpzrIA2KYhQ0NXgKAHs-obO-WGBd5buEQeqZzTvbP25fL9JTOHZOjxd7-iqblMoHku2CbrzPgpP8Ic7CWHb8f1fuwqpwTuGmTKxeTJ0nHkWHt3Tr7DbQ-aK66cWIrUYFPjGnURiCd0MStnjL1BXkNrilhmrLbufamtR5J2zmSFFgGZ4eVkXU2Duh2vCk9bIrrczS44Xi9MjWpD4Kv8-UI-7ypqUVOjcrjKacBjw6zhDcW4wfjTj9rDG5NaXD6nsNV0WSmZoXPL8jkwvTdQaS-r9j8UfBkCGbyK-JgxYS8BQaeLnVLzjOqW6WyTgKU2XUZh3XvI2cimCGW5rk0ZegmCk3Ygw5ulap-Y9bdvw0wGjypi-6_d-ZkRcqLQ5Z5r-mGYxdwKdBDoBx3ERM5x4PKG3awadibSkniVC4vUkEH87XEJxnetRpw-adp2pSJo_9AfueXuct6-LIIapjg2XvG8QID1DutoSvGeXPoZ70dS1nrK420bjfRBbaYPJLjAqO_HstKTZfbJT2TdfSpk_7dyZan5Zk1b-Nnbp_XnFpGKk_YC1ReCMyuc3B0nVU814wjALh-Fkiy_rBEg95f7WTHsLvgZVTgfSW95P_SOKrJA7ocCjQ9KlqknkGE7w5rK73uN_IEVRkAxjG-GC4S1aIoUz5txJN7xYAtKDm07VY_wbpGrv2etzdwHgXbhjiYBwVEgFyVxLATq2uNPVhoEXPfqvAbioKcGmQvDPBsxrBzEA_zt9HoT4ZqbG7jl1PHlevr5GLAjGMq0k9PznpeHS3nc0KIxdjzRnpzS5SvjSF4ke2_zFPNuSclNJPqqPrPG2iBiP0Eum7Fgdp9YdYHmJ9bCBsZab-jVcttplbfSiktCyNMJhxm8bcsV91rwY7XXBurotMNvct0Bvhlf5dTGhSkeWDRntFtlrbf86YWK272tAXY636tLL92s3AE2Y1HIfyYDYDYr7y7765I0LGJ2VPUy85B6NT0QR6dA-PAp9uCOD5Lsu8hBlu3TTI4JezJscY2VhQ0JwzYZmsKe4TOEHGJf6lGnYO7i8uWSBPt_ZpK3FhtKpk0g91CP1v1RG8zBlw7gdtjKsmS-LQb4LLDHt2yClWrpuP50q3dQ_Twp0TKhomvSJ_fXUX3Ssa33mkeXL6S61oknJlGhf2GKJ--CUXJhI6y3fnhqn0HRmUpASUTVuiEKBSdeuRNhe6owypPNr_yivZ5D2QToJZoSpWxBLvHiEgXoMUYuxdVPxgnuk9eawXT0pc3a6EgvZ4IRaNmIIs74UupBs-FKmamgHxrVvfENO3X2ZN9CCRudFy14vEwLAB-yPDKPIiMfFzj5Qfrgq93tQFBgM0VkG_EI6ockl2C5pZYG0qH8Hm6UXlNnba8FHwYgpx0D70YE-iXTXY81a1OHNmJbA6n9_ahbquNbgQ83w25qp9Gra2admdlqtrMVoC1YNPW9nFA-onEmRT67RFTpRQ5XT8vcM-01dgIiUIKWMsjUktjnF9p3dft3TeaXK4GrfoUbGLpm92-0_ghZuW6V4syUOYkEvPnvlvBXljmUCJhzNl7vCt3_GkDqMslsRA-I5AjGEFzEWnoE6gI0VKpXSarD7ra4eci3174b7Nc5pvqax7XOlc0s_-tNAoYqffEcvhR0lpZjqI=", + "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", "id": "", "type": "compaction" } @@ -1591,10 +1591,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 190, + "completion_tokens": 235, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 133, "time_to_first_token": 0, - "tokens": 323 + "tokens": 368 } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json index 034953fc9..5001d0fd8 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json @@ -1036,7 +1036,7 @@ { "annotations": [], "logprobs": [], - "text": "PARIS.", + "text": "PARIS", "type": "output_text" } ], @@ -1064,7 +1064,7 @@ "model": "gpt-4o-mini-2024-07-18", "moderation": null, "object": "response", - "output_text": "PARIS.", + "output_text": "PARIS", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1113,12 +1113,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 3, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 } } ], @@ -1256,7 +1256,7 @@ { "annotations": [], "logprobs": [], - "text": "RESPONSE STREAM.", + "text": "RESPONSE STREAM", "type": "output_text" } ], @@ -1330,12 +1330,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 5, + "completion_tokens": 4, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 18 + "tokens": 17 } } ], @@ -1469,7 +1469,7 @@ { "annotations": [], "logprobs": [], - "text": "PARTIAL.", + "text": "PARTIAL", "type": "output_text" } ], @@ -1542,12 +1542,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 4, + "completion_tokens": 3, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 13, "time_to_first_token": 0, - "tokens": 17 + "tokens": 16 } } ], @@ -1576,10 +1576,10 @@ "annotations": [], "logprobs": [], "parsed": { - "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + "reasoning": "20 plus 4 equals 24.", "value": 24 }, - "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", "type": "output_text" } ], @@ -1608,10 +1608,10 @@ "moderation": null, "object": "response", "output_parsed": { - "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + "reasoning": "20 plus 4 equals 24.", "value": 24 }, - "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", "parallel_tool_calls": true, "presence_penalty": 0, "previous_response_id": null, @@ -1679,12 +1679,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 30, + "completion_tokens": 17, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 46, "time_to_first_token": 0, - "tokens": 76 + "tokens": 63 } } ], @@ -1739,7 +1739,7 @@ "type": "message" }, { - "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", "id": "", "type": "compaction" } @@ -1754,12 +1754,12 @@ }, "metrics": { "completion_reasoning_tokens": 0, - "completion_tokens": 220, + "completion_tokens": 204, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 133, "time_to_first_token": 0, - "tokens": 353 + "tokens": 337 } } ], diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt index 6475d02dc..29e5058b6 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt @@ -921,7 +921,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARIS.", + │ "text": "PARIS", │ "type": "output_text" │ } │ ], @@ -949,7 +949,7 @@ span_tree: │ "model": "gpt-4o-mini-2024-07-18", │ "moderation": null, │ "object": "response", - │ "output_text": "PARIS.", + │ "output_text": "PARIS", │ "parallel_tool_calls": true, │ "presence_penalty": 0, │ "previous_response_id": null, @@ -998,12 +998,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 3, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 16 │ } ├── openai-responses-with-response-operation │ metadata: { @@ -1125,7 +1125,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "RESPONSE STREAM.", + │ "text": "RESPONSE STREAM", │ "type": "output_text" │ } │ ], @@ -1199,12 +1199,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 5, + │ "completion_tokens": 4, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 18 + │ "tokens": 17 │ } ├── openai-responses-stream-operation │ metadata: { @@ -1322,7 +1322,7 @@ span_tree: │ { │ "annotations": [], │ "logprobs": [], - │ "text": "PARTIAL.", + │ "text": "PARTIAL", │ "type": "output_text" │ } │ ], @@ -1395,12 +1395,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, + │ "completion_tokens": 3, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 13, │ "time_to_first_token": 0, - │ "tokens": 17 + │ "tokens": 16 │ } ├── openai-responses-parse-operation │ metadata: { @@ -1421,10 +1421,10 @@ span_tree: │ "annotations": [], │ "logprobs": [], │ "parsed": { - │ "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + │ "reasoning": "20 plus 4 equals 24.", │ "value": 24 │ }, - │ "text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", │ "type": "output_text" │ } │ ], @@ -1453,10 +1453,10 @@ span_tree: │ "moderation": null, │ "object": "response", │ "output_parsed": { - │ "reasoning": "The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.", + │ "reasoning": "20 plus 4 equals 24.", │ "value": 24 │ }, - │ "output_text": "{\"reasoning\":\"The sum of 20 and 4 is straightforward addition: 20 + 4 equals 24.\",\"value\":24}", + │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", │ "parallel_tool_calls": true, │ "presence_penalty": 0, │ "previous_response_id": null, @@ -1524,12 +1524,12 @@ span_tree: │ } │ metrics: { │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 30, + │ "completion_tokens": 17, │ "prompt_cache_write_tokens": 0, │ "prompt_cached_tokens": 0, │ "prompt_tokens": 46, │ "time_to_first_token": 0, - │ "tokens": 76 + │ "tokens": 63 │ } └── openai-responses-compact-operation metadata: { @@ -1576,7 +1576,7 @@ span_tree: "type": "message" }, { - "encrypted_content": "gAAAAABqXyazVfGnxSVNeX_1-zzgt3J_6njbfKWC7TDkctLEMbzGx0qtH23Gqj45G9KWqoJStbxxwFUuPV6VkzwaXzb8cBveQFofzPIRzoQa3wlZ3blX-7QqHLzkgo-_R0BdYwstB4C2WuoySegScvT1Vg7ZYAVlUatGKCmM_8kXc_ytoNbGfI8ElX_5ZjJxnOXeSqpeb989gLnNmS83e0e-AW2esgoRKT1_SeveBLRIRVfsaEKmnUfMQsp0geEYePq3y0rs-GQNIpf4EZ1iL_OIe3BY_AXEa4dmduc3iwysF1A2HBlGhiVpjn4gJS4mqkQIFw72Yupsq5oxj-7JZE977AP6UgaATiVzsLJ5wGjAzkNQEfKQTFXHo3E8wQlgGqXodjnGfB1j0hTNSwtO1ES2suLM5JAmB_5Mcq63qTpLdux5ZRFm5sVGXC0U3j_1g2tuoOLsHlGAkJqgaJ5-ICCPKSVcWrIHpnSXwiaaDOXOtrxzUb8YGJLlMWq34cYv8mwcwksKTcM1U9gNRTUjH0GrvKk7GENB0cHRDH5u4e7AjopSLwt_8tkk7sdPnrpq49Wp5Y81Fzg7W5yCtqvleuoNpkx6Fi5npizu1I_tXkPVhAtmNEnINcQh943a3-MWxxNczCARlkwsCVSEL1xgVLKruz4QlrlbkK5gX6uo9dm7kunbE2BZ7EeAQLl695kLSPFc1WOhYtegOpwg8U0ZNmEd1j8qsKlMCyJfZT61uOWZgY2ZeHlzGffBdDL1mFZ0c-Gt1VUAnB7iTFmViYIQNTxqK96lf3OfIZh--5vXkUTG8uUwH_19a8wRVQHxdEzi_sp91WjbyHc3OpzqWOucz9XPvW7RuSTkH0SkDIvDufGHH9kHxcxOIqxjgq9Dko05TriWbcycvpERwENem5w28pRrkFeTuT3-TnMK-MSzFG9TyGhca0HvZISCnq-frPEYX6v0zkZ85Zb073Im5GJh7nSQTz3vOtlADQOEoQig75hit2ZJj7iscc6zIMfQt9WBqHs8P-R3ZImJtSVbgcxRA1aoQz9hzqwqL3QGGjiFDWn2l04Y4fVZDC3XKmOY38hvOQas0BeiEvlyIBhazMHWdoeIxXVCOmjlOu4Es0wugH_Ji313bww0sujV0PY5zZYs1axylpfrhIgc1qRk2PTd-VJghDO2htsu_RUBtc3gg_2MQbdGiTolw9MJD1cJcl2G--fJo7Ch3enKiR0JMA1u5WEaK9KxV_9dZo357w6VRER2cK8s7XpaJhzN94T1_xFalhGa3L73quAvTM2wWwGfA8Njsu5g63_JeyX2n8uaCdqvLHvlo9MBhaOGSX1xdeIX_liFVulUfvElWXpfmldjUbEVZj8G0P4VdMapxVvdgUjoNYDj0KGVwFPZjz2xopnTNsrbPS2piB5dz0pQjHTTREHbBCPxV4oETq_nHwTcAyJSnKwF7Xh60tiDp-sBq0d896hsE84I5BL2E0Ceo-JBk9E2os7sa4lb4Ps97sa03VoBcJaU8uoZycNmpjL73aldwJeMGCvwA4qkB2aO4vUHs-KCgeJb2q4jm6HLxaBA5OZvXqoTWq26LMvtyhy9oEOiSvIS6PHMvOukmtCGVjAQ9C2Ah9x-3FydPa56apBx9S-a4Fd8ZJJtdJIDR8tOKS3qN-PsAoQyYOqESuvELZhy5IawwvPspSUX8hfmhcH7wN__XJuKb0n8_Qn-bqeL5L-hB2ZvJ8Xe_SwuSOVkTvnkgNDrQoKtciOcodyfiIVxVxKUD_C6L-Zf_eSPpHYijFr5CQJy-ln_bKiZsQXMzcOIYbDZvXWvWuGeuxHvhWC6LDb7Qdn1rKxuA3GGuS1IyINT--KPBbpWSiRBFGxLvfN5wtCbJGuK2xCvGS4oAI3T0pshn-ml2s403vNec2tUgbfXMvH1AW8kbO6s14QyBj5m75_0X9pCiaRKMT98UrnPynY-itCWhSWmNlayGYVJl9AXA-0fgSRl8Jb343tB1wO5KG8eFRyNYFnWGGrQwEfyT71ORHyd-JcfgiFY5AziXVAfK6LDierNi6vGNTzzO0btEUHFBZd-hIxN7a_WmpzrUwJ67M6wQD1trsYACjEhpWJsCfmpwqj9vrIopHwteZb010uM_WRmsSPQn0m37g24PZcoo-G6KqS-Mf0kLzUj3rwbt8-YxwhNvwCkmjTR6OR4eQN9URYZlXrpZXX5WHIWd8OE9dAlodkvu4Ur0WbX01O42Tp_SGmjgy1orEbXUpOYAsKmArQwgsn1LRmxY6CIF4dqJPUL1wsxCiyh893SRwycW5w2b6DXwMv-mvzOLUAYi_XRduCs71FB6-vPYaO-Bq3CdJo45Ydi3CQ4yjrB7hRSOxd2w6jwhLUZZfIjVfiqfU0-yL5EmjnRSC9-Qh3Rw5KF_nULKG-uWGoxQk394XAu654jj9Vt4Ak5xWtOsczw1UegcNheFvt37N8hpmjLCz7rYu2Umi5U5md7jb8NqKIoVzc5V2Y4b86cPp_x3eJYWzhgqdMFdR06qN7-YFsgVu80ZhF2YVwTVFebeFMv8RE0K1bhvpGrS9zLWul5KANmKLAUxLSFttnAC5ukwmfZBZtOzEdhTa8J5J1dQQeBWLiCtblIc6vqP115KvU1ZunIRjwc3zcFBxmQopsYI5Qw6aRpeKK-RIe_sc2gHQQ5yy3gVVUl-kK2T2Etw0a4_CXyQnUVAiDOLk8RDWsKjS3Oz1WF0FcAJs0DnproHNVYt5tEehaPrFELJOXkn5Lt8WSDouhxmQJrfEIRnQWerR9rGz-GMF27ELiY9k6inSG7tB42Eb-ZRF832HMi9VVZufaUCYxEcfPanR8IckU-JPerTtCdO2ATxVefyKcRiX0hWKEU7AH-l1Y8UqSU91ZC1NORUS-z6mumQ909FxRepme15tRrYqVJ5U0dQ9xkg1TLwdK9jYTyVzN8D8Z3ibLOSwVJXxt-gcixA3Z9_ojyOdDQqQ1rO7pNXOraD6IMSKq9NaYCy8GvjRd6Rd6UUqC5bjK7DR8LpzsfCmh9OWzdoV1dszEDQohqak7E9ZIiT8J32DDzb7_lcvZBtVN0xNEF8DwwDxhQcc3MpTU1t9Ey24SfmbC6vh4DC4mqEw7wjqYpmplturczVNBhg4unk5MYXOsbAU9VN37hlFID8z6ohBdn0TMWaEbcFy8EjKsU5Nf7pv0cAoGvZ48eEOgCtnYCEPxpBWAkDWftNFupjE1tDjxdHL6NU_sNBQoMe61SvuzWGg5E46enkGUU05LbMzCG9Z204mBIk1F_XHbQ36DVmgBubjTJQMuK1UhzhVk6Qcy89x-6yMUoOVLTf_xk5362RSG_xxAmNHvehWHMHg==", + "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", "id": "", "type": "compaction" } @@ -1591,10 +1591,10 @@ span_tree: } metrics: { "completion_reasoning_tokens": 0, - "completion_tokens": 220, + "completion_tokens": 204, "prompt_cache_write_tokens": 0, "prompt_cached_tokens": 0, "prompt_tokens": 133, "time_to_first_token": 0, - "tokens": 353 + "tokens": 337 } diff --git a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs index d17abf4fb..5673c8d9a 100644 --- a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs @@ -7,7 +7,7 @@ import { runOperation, runTracedScenario, } from "../../helpers/provider-runtime.mjs"; -import { completeOpenAIBatch, createOpenAIBatch } from "braintrust"; +import { completeOpenAIBatchTrace, startOpenAIBatchTrace } from "braintrust"; const OPENAI_MODEL = "gpt-4o-mini-2024-07-18"; const EMBEDDING_MODEL = "text-embedding-3-small"; @@ -650,7 +650,7 @@ export async function runOpenAIInstrumentationScenario(options) { }), ) .join("\n"); - const created = await createOpenAIBatch({ + const created = await startOpenAIBatchTrace({ client: batchFixtureClient, inputFile: { id: "file_batch_e2e_fixture" }, input, @@ -712,7 +712,7 @@ export async function runOpenAIInstrumentationScenario(options) { }), ), ); - await completeOpenAIBatch({ + await completeOpenAIBatchTrace({ batch: completed, input, // Batch results are not guaranteed to preserve input order. diff --git a/js/src/exports.ts b/js/src/exports.ts index ccca0f2d6..bd20c477f 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -129,7 +129,10 @@ export { export { registerSandbox } from "./sandbox"; -export { completeOpenAIBatch, createOpenAIBatch } from "./openai-batch"; +export { + completeOpenAIBatchTrace, + startOpenAIBatchTrace, +} from "./openai-batch"; // Internal isomorph layer for platform-specific implementations import _internalIso from "./isomorph"; diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts index da44afcc6..6174c64ad 100644 --- a/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts @@ -7,7 +7,10 @@ import { withCurrent, } from "../../logger"; import { configureNode } from "../../node/config"; -import { completeOpenAIBatch, createOpenAIBatch } from "../../openai-batch"; +import { + completeOpenAIBatchTrace, + startOpenAIBatchTrace, +} from "../../openai-batch"; import type { OpenAIBatchCreateInputParams, OpenAIBatchJSONL, @@ -44,12 +47,12 @@ const chatInput = [ }, ]; -async function createTestBatch(args: { +async function startTestBatchTrace(args: { input: OpenAIBatchJSONL; inputFile: OpenAIFileLike; params: OpenAIBatchCreateInputParams; }) { - return await createOpenAIBatch({ + return await startOpenAIBatchTrace({ ...args, client: { batches: { @@ -130,7 +133,7 @@ describe("OpenAI Batch instrumentation", () => { completion_window: "24h" as const, metadata: { user: "value" }, }; - const params = await createTestBatch({ + const params = await startTestBatchTrace({ inputFile, input: input.map((record) => JSON.stringify(record)).join("\n"), params: sourceParams, @@ -181,7 +184,7 @@ describe("OpenAI Batch instrumentation", () => { }); it("does not start spans when OpenAI returns a different batch context", async () => { - const batch = await createOpenAIBatch({ + const batch = await startOpenAIBatchTrace({ inputFile: { id: "file_mismatched_response" }, input: [chatInput[0]], params: { @@ -214,11 +217,11 @@ describe("OpenAI Batch instrumentation", () => { }, }; - const first = await createTestBatch(args); + const first = await startTestBatchTrace(args); const firstRows = (await backgroundLogger.drain()) as Array< Record >; - const second = await createTestBatch(args); + const second = await startTestBatchTrace(args); const secondRows = (await backgroundLogger.drain()) as Array< Record >; @@ -233,7 +236,7 @@ describe("OpenAI Batch instrumentation", () => { }); it("signs and verifies batch context with WebCrypto", async () => { - const params = await createTestBatch({ + const params = await startTestBatchTrace({ inputFile: { id: "file_webcrypto" }, input: [chatInput[0]], params: { @@ -244,7 +247,7 @@ describe("OpenAI Batch instrumentation", () => { expect(params.metadata).toHaveProperty(BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY); await backgroundLogger.drain(); - await completeOpenAIBatch({ + await completeOpenAIBatchTrace({ batch: { id: "batch_webcrypto", endpoint: params.endpoint, @@ -273,7 +276,7 @@ describe("OpenAI Batch instrumentation", () => { propagatedEvent: { metadata: { secret: "do-not-send" } }, }); const batch = await withCurrent(parent, () => - createTestBatch({ + startTestBatchTrace({ inputFile: { id: "file_minimal_parent" }, input: [chatInput[0]], params: { @@ -302,7 +305,7 @@ describe("OpenAI Batch instrumentation", () => { }; await expect( - completeOpenAIBatch({ + completeOpenAIBatchTrace({ batch, input: chatInput, outputFile: Promise.reject(new Error("output unavailable")), @@ -314,7 +317,7 @@ describe("OpenAI Batch instrumentation", () => { }); it("completes successful and failed spans from resumable files", async () => { - const params = await createTestBatch({ + const params = await startTestBatchTrace({ inputFile: { id: "file_input" }, input: chatInput, params: { @@ -381,7 +384,7 @@ describe("OpenAI Batch instrumentation", () => { batch.completed_at = completedAt; await expect( - completeOpenAIBatch({ + completeOpenAIBatchTrace({ batch, input: chatInput, outputFile: Promise.resolve(outputResponse), @@ -420,7 +423,7 @@ describe("OpenAI Batch instrumentation", () => { "failed", ); - await completeOpenAIBatch({ + await completeOpenAIBatchTrace({ batch, input: chatInput, outputFile: Promise.resolve(chunkedJSONLResponse(outputRecords)), @@ -433,7 +436,7 @@ describe("OpenAI Batch instrumentation", () => { }); it("leaves a completed batch pending until all results are available", async () => { - const batch = await createTestBatch({ + const batch = await startTestBatchTrace({ inputFile: { id: "file_incomplete" }, input: chatInput, params: { @@ -443,7 +446,7 @@ describe("OpenAI Batch instrumentation", () => { }); await backgroundLogger.drain(); - await completeOpenAIBatch({ + await completeOpenAIBatchTrace({ batch: { ...batch, status: "completed", @@ -471,7 +474,7 @@ describe("OpenAI Batch instrumentation", () => { }); it("rejects completion input that does not match the signed batch input", async () => { - const params = await createTestBatch({ + const params = await startTestBatchTrace({ inputFile: { id: "file_mismatched" }, input: chatInput, params: { @@ -481,7 +484,7 @@ describe("OpenAI Batch instrumentation", () => { }); await backgroundLogger.drain(); - await completeOpenAIBatch({ + await completeOpenAIBatchTrace({ batch: { id: "batch_mismatched", endpoint: params.endpoint, @@ -514,7 +517,7 @@ describe("OpenAI Batch instrumentation", () => { }); it("falls back from invalid provider terminal timestamps", async () => { - const params = await createTestBatch({ + const params = await startTestBatchTrace({ inputFile: { id: "file_invalid_timestamp" }, input: [chatInput[0]], params: { @@ -525,7 +528,7 @@ describe("OpenAI Batch instrumentation", () => { await backgroundLogger.drain(); const beforeCompletion = Date.now() / 1000; - await completeOpenAIBatch({ + await completeOpenAIBatchTrace({ batch: { id: "batch_invalid_timestamp", endpoint: params.endpoint, @@ -568,7 +571,7 @@ describe("OpenAI Batch instrumentation", () => { body: { model: "gpt-test", input: "hello" }, }; } - const params = await createTestBatch({ + const params = await startTestBatchTrace({ inputFile: { id: "file_responses" }, input: input(), params: { endpoint: "/v1/responses", completion_window: "24h" }, @@ -600,7 +603,7 @@ describe("OpenAI Batch instrumentation", () => { const completedAt = Date.now() / 1000 + 60; batch.completed_at = completedAt; - await completeOpenAIBatch({ + await completeOpenAIBatchTrace({ batch, input: input(), outputFile, @@ -629,7 +632,7 @@ describe("OpenAI Batch instrumentation", () => { }); expect(responseSpan?.metrics).not.toHaveProperty("time_to_first_token"); - await completeOpenAIBatch({ + await completeOpenAIBatchTrace({ batch, input: input(), outputFile, @@ -639,7 +642,7 @@ describe("OpenAI Batch instrumentation", () => { }); it("ends a result span when output processing throws", async () => { - const params = await createTestBatch({ + const params = await startTestBatchTrace({ inputFile: { id: "file_invalid_output" }, input: [ { @@ -662,7 +665,7 @@ describe("OpenAI Batch instrumentation", () => { }, }); const completedAt = Date.now() / 1000 + 60; - await completeOpenAIBatch({ + await completeOpenAIBatchTrace({ batch: { id: "batch_invalid_output", endpoint: params.endpoint, @@ -704,7 +707,7 @@ describe("OpenAI Batch instrumentation", () => { inputReads++; yield* chatInput; } - const params = await createTestBatch({ + const params = await startTestBatchTrace({ inputFile: { id: "file_input" }, input: input(), params: { diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts index fcd81e434..14f855199 100644 --- a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts @@ -20,11 +20,11 @@ import { type SpanComponentsV4Data, } from "../../../util/span_identifier_v4"; import type { - CompleteOpenAIBatchArgs, - CreateOpenAIBatchArgs, + CompleteOpenAIBatchTraceArgs, OpenAIBatchCreateParams, OpenAIBatchJSONL, OpenAIBatchLike, + StartOpenAIBatchTraceArgs, } from "../../openai-batch-types"; import { BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY } from "./openai-batch-constants"; import { openAIChannels } from "./openai-channels"; @@ -67,7 +67,7 @@ type BatchInputRecord = { }; type OpenAIBatchResultFile = - CompleteOpenAIBatchArgs["outputFile"]; + CompleteOpenAIBatchTraceArgs["outputFile"]; type PreparedBatch = { context: BatchContext; @@ -279,7 +279,7 @@ async function exportMinimalParent( } async function prepareCreateParams( - args: CreateOpenAIBatchArgs, + args: StartOpenAIBatchTraceArgs, parent: ReturnType, startTime: number, ): Promise { @@ -674,8 +674,8 @@ async function startPreparedBatch( } } -export const interceptOpenAIBatchCreate: Parameters< - typeof openAIChannels.batchesCreate.intercept +export const interceptOpenAIBatchTraceStart: Parameters< + typeof openAIChannels.batchesStartTrace.intercept >[0] = async (target, thisArg, args) => { const parent = getSpanParentObject(); const startTime = getCurrentUnixTimestamp(); @@ -997,7 +997,7 @@ function terminalEndTime( } async function completeBatch( - args: CompleteOpenAIBatchArgs, + args: CompleteOpenAIBatchTraceArgs, ): Promise { const { batch, webhook } = args; const batchId = read(batch, "id"); @@ -1107,8 +1107,8 @@ async function completeBatch( task.end({ endTime }); } -export const interceptOpenAIBatchComplete: Parameters< - typeof openAIChannels.batchesComplete.intercept +export const interceptOpenAIBatchTraceComplete: Parameters< + typeof openAIChannels.batchesCompleteTrace.intercept >[0] = (target, thisArg, args) => Promise.resolve(Reflect.apply(target, thisArg, args)).then(async (batch) => { try { diff --git a/js/src/instrumentation/plugins/openai-channels.ts b/js/src/instrumentation/plugins/openai-channels.ts index 1d27b0366..b966acd1c 100644 --- a/js/src/instrumentation/plugins/openai-channels.ts +++ b/js/src/instrumentation/plugins/openai-channels.ts @@ -18,9 +18,9 @@ import type { OpenAIResponseStreamEvent, } from "../../vendor-sdk-types/openai"; import type { - CompleteOpenAIBatchArgs, - CreateOpenAIBatchArgs, + CompleteOpenAIBatchTraceArgs, OpenAIBatchLike, + StartOpenAIBatchTraceArgs, } from "../../openai-batch-types"; type OpenAIChatSpanInfo = NonNullable["span_info"]>; @@ -36,21 +36,21 @@ type OpenAIResponsesChannelExtras = OpenAIChannelExtras; export const openAIChannels = defineChannels( "openai", { - batchesCreate: channel< - [CreateOpenAIBatchArgs], + batchesStartTrace: channel< + [StartOpenAIBatchTraceArgs], OpenAIBatchLike, OpenAIChannelExtras >({ - channelName: "batches.create-resumable", + channelName: "batches.start-trace", kind: "async", }), - batchesComplete: channel< - [CompleteOpenAIBatchArgs], + batchesCompleteTrace: channel< + [CompleteOpenAIBatchTraceArgs], OpenAIBatchLike, OpenAIChannelExtras >({ - channelName: "batches.complete", + channelName: "batches.complete-trace", kind: "async", }), diff --git a/js/src/instrumentation/plugins/openai-plugin.ts b/js/src/instrumentation/plugins/openai-plugin.ts index 0ee68a344..f20416e8c 100644 --- a/js/src/instrumentation/plugins/openai-plugin.ts +++ b/js/src/instrumentation/plugins/openai-plugin.ts @@ -15,8 +15,8 @@ import { processImagesInOutput, } from "./openai-span-data"; import { - interceptOpenAIBatchComplete, - interceptOpenAIBatchCreate, + interceptOpenAIBatchTraceComplete, + interceptOpenAIBatchTraceStart, } from "./openai-batch-instrumentation"; import { BRAINTRUST_CACHED_STREAM_METRIC, @@ -47,8 +47,12 @@ export class OpenAIPlugin extends BasePlugin { protected onEnable(): void { this.unsubscribers.push( - openAIChannels.batchesCreate.intercept(interceptOpenAIBatchCreate), - openAIChannels.batchesComplete.intercept(interceptOpenAIBatchComplete), + openAIChannels.batchesStartTrace.intercept( + interceptOpenAIBatchTraceStart, + ), + openAIChannels.batchesCompleteTrace.intercept( + interceptOpenAIBatchTraceComplete, + ), ); // Chat Completions - supports streaming diff --git a/js/src/openai-batch-types.ts b/js/src/openai-batch-types.ts index f21a0540b..700ff8322 100644 --- a/js/src/openai-batch-types.ts +++ b/js/src/openai-batch-types.ts @@ -66,14 +66,14 @@ export interface OpenAIBatchWebhookEvent { [key: string]: unknown; } -export interface CreateOpenAIBatchArgs { +export interface StartOpenAIBatchTraceArgs { client: OpenAIBatchClientLike; inputFile: OpenAIFileLike; input: OpenAIBatchJSONL; params: OpenAIBatchCreateInputParams; } -export interface CompleteOpenAIBatchArgs { +export interface CompleteOpenAIBatchTraceArgs { batch: TBatch; input: OpenAIBatchResultFile; outputFile?: OpenAIBatchResultFile; diff --git a/js/src/openai-batch.ts b/js/src/openai-batch.ts index 510997816..db587ec7e 100644 --- a/js/src/openai-batch.ts +++ b/js/src/openai-batch.ts @@ -1,19 +1,19 @@ import { openAIChannels } from "./instrumentation/plugins/openai-channels"; import type { - CompleteOpenAIBatchArgs, - CreateOpenAIBatchArgs, + CompleteOpenAIBatchTraceArgs, OpenAIBatchLike, + StartOpenAIBatchTraceArgs, } from "./openai-batch-types"; /** - * Create an OpenAI Batch and start pending Braintrust spans for its requests. + * Start a trace for a new OpenAI Batch, with pending spans for its requests. * * Supported endpoints are `/v1/chat/completions` and `/v1/responses`. */ -export async function createOpenAIBatch( - args: CreateOpenAIBatchArgs, +export async function startOpenAIBatchTrace( + args: StartOpenAIBatchTraceArgs, ): Promise { - const batch = await openAIChannels.batchesCreate.invoke( + const batch = await openAIChannels.batchesStartTrace.invoke( async (input) => await input.client.batches.create({ ...input.params, @@ -28,14 +28,14 @@ export async function createOpenAIBatch( } /** - * Complete pending OpenAI Batch spans from caller-supplied Batch API data. + * Complete an OpenAI Batch trace from caller-supplied Batch API data. * * This helper performs no OpenAI API requests. Pass the Responses or promises * returned by `client.files.content()` for the input and result files, JSONL * strings, or iterables of parsed records. */ -export async function completeOpenAIBatch( - args: CompleteOpenAIBatchArgs, +export async function completeOpenAIBatchTrace( + args: CompleteOpenAIBatchTraceArgs, ): Promise { const input = Promise.resolve(args.input); const outputFile = @@ -52,7 +52,7 @@ export async function completeOpenAIBatch( void outputFile?.catch(() => undefined); void errorFile?.catch(() => undefined); - const batch = await openAIChannels.batchesComplete.invoke( + const batch = await openAIChannels.batchesCompleteTrace.invoke( async (input) => input.batch, undefined, [{ ...args, input, outputFile, errorFile }], From 1d96a06d25b46ef283ebf4ff4dd0bdd2e63d7dfb Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:28:20 +0000 Subject: [PATCH 4/6] Update PR #2369 --- .../openai-instrumentation/scenario.impl.mjs | 10 +-- .../openai-batch-instrumentation.test.ts | 71 +++++++++---------- .../plugins/openai-batch-instrumentation.ts | 50 +++++-------- .../plugins/openai-channels.ts | 5 +- js/src/openai-batch-types.ts | 23 +++--- js/src/openai-batch.ts | 32 ++++----- 6 files changed, 80 insertions(+), 111 deletions(-) diff --git a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs index 5673c8d9a..9dc04604a 100644 --- a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs @@ -650,8 +650,7 @@ export async function runOpenAIInstrumentationScenario(options) { }), ) .join("\n"); - const created = await startOpenAIBatchTrace({ - client: batchFixtureClient, + const batchParams = await startOpenAIBatchTrace({ inputFile: { id: "file_batch_e2e_fixture" }, input, params: { @@ -659,6 +658,7 @@ export async function runOpenAIInstrumentationScenario(options) { endpoint: "/v1/chat/completions", }, }); + const created = await batchFixtureClient.batches.create(batchParams); const completedAt = Date.now() / 1000 + 60; const completed = { @@ -714,7 +714,7 @@ export async function runOpenAIInstrumentationScenario(options) { ); await completeOpenAIBatchTrace({ batch: completed, - input, + inputFile: input, // Batch results are not guaranteed to preserve input order. outputFile, errorFile, @@ -724,8 +724,8 @@ export async function runOpenAIInstrumentationScenario(options) { data: { id: completed.id }, }, }); - if ((await outputFile).bodyUsed || (await errorFile).bodyUsed) { - throw new Error("Expected batch result responses to remain unread"); + if (!(await outputFile).bodyUsed || !(await errorFile).bodyUsed) { + throw new Error("Expected batch result responses to be consumed"); } }); }, diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts index 6174c64ad..e9560bbb1 100644 --- a/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts @@ -52,18 +52,12 @@ async function startTestBatchTrace(args: { inputFile: OpenAIFileLike; params: OpenAIBatchCreateInputParams; }) { - return await startOpenAIBatchTrace({ - ...args, - client: { - batches: { - create: async (params) => ({ - id: `batch_${args.inputFile.id}`, - ...params, - status: "validating", - }), - }, - }, - }); + const params = await startOpenAIBatchTrace(args); + return { + id: `batch_${args.inputFile.id}`, + ...params, + status: "validating", + }; } function chunkedJSONLResponse( @@ -183,28 +177,25 @@ describe("OpenAI Batch instrumentation", () => { ).not.toHaveProperty("user"); }); - it("does not start spans when OpenAI returns a different batch context", async () => { - const batch = await startOpenAIBatchTrace({ - inputFile: { id: "file_mismatched_response" }, + it("prepares parameters without calling the OpenAI API", async () => { + const params = await startOpenAIBatchTrace({ + inputFile: { id: "file_prepare" }, input: [chatInput[0]], params: { endpoint: "/v1/chat/completions", completion_window: "24h", }, - client: { - batches: { - create: async (params) => ({ - ...params, - id: "batch_mismatched_response", - endpoint: "/v1/responses", - status: "validating", - }), - }, - }, }); - expect(batch.endpoint).toBe("/v1/responses"); - expect(await backgroundLogger.drain()).toEqual([]); + expect(params).toMatchObject({ + input_file_id: "file_prepare", + endpoint: "/v1/chat/completions", + completion_window: "24h", + metadata: { + [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: expect.any(String), + }, + }); + expect(await backgroundLogger.drain()).toHaveLength(2); }); it("keeps separate batch operations distinct when they reuse an input file", async () => { @@ -256,7 +247,7 @@ describe("OpenAI Batch instrumentation", () => { completed_at: 200, metadata: params.metadata, }, - input: [chatInput[0]], + inputFile: [chatInput[0]], outputFile: [ { custom_id: "ok", @@ -307,7 +298,7 @@ describe("OpenAI Batch instrumentation", () => { await expect( completeOpenAIBatchTrace({ batch, - input: chatInput, + inputFile: chatInput, outputFile: Promise.reject(new Error("output unavailable")), errorFile: Promise.reject(new Error("errors unavailable")), }), @@ -380,20 +371,22 @@ describe("OpenAI Batch instrumentation", () => { errorRecords, waitForBothReaders, ); + const inputResponse = chunkedJSONLResponse(chatInput); const completedAt = Date.now() / 1000 + 60; batch.completed_at = completedAt; await expect( completeOpenAIBatchTrace({ batch, - input: chatInput, + inputFile: Promise.resolve(inputResponse), outputFile: Promise.resolve(outputResponse), errorFile: Promise.resolve(errorResponse), }), ).resolves.toBe(batch); expect(readersStarted).toBe(2); - expect(outputResponse.bodyUsed).toBe(false); - expect(errorResponse.bodyUsed).toBe(false); + expect(inputResponse.bodyUsed).toBe(true); + expect(outputResponse.bodyUsed).toBe(true); + expect(errorResponse.bodyUsed).toBe(true); const rows = (await backgroundLogger.drain()) as Array>; expect(rows).toHaveLength(3); expect(rows.map((row) => row.id).sort()).toEqual(pendingIds); @@ -425,7 +418,7 @@ describe("OpenAI Batch instrumentation", () => { await completeOpenAIBatchTrace({ batch, - input: chatInput, + inputFile: chatInput, outputFile: Promise.resolve(chunkedJSONLResponse(outputRecords)), errorFile: Promise.resolve(chunkedJSONLResponse(errorRecords)), }); @@ -453,7 +446,7 @@ describe("OpenAI Batch instrumentation", () => { completed_at: 400, request_counts: { completed: 1, failed: 1, total: 2 }, }, - input: chatInput, + inputFile: chatInput, outputFile: [ { custom_id: "ok", @@ -494,7 +487,7 @@ describe("OpenAI Batch instrumentation", () => { request_counts: { completed: 1, failed: 1, total: 2 }, metadata: params.metadata, }, - input: chatInput.map((record) => ({ + inputFile: chatInput.map((record) => ({ ...record, body: { ...record.body, @@ -538,7 +531,7 @@ describe("OpenAI Batch instrumentation", () => { request_counts: { completed: 1, failed: 0, total: 1 }, metadata: params.metadata, }, - input: [chatInput[0]], + inputFile: [chatInput[0]], outputFile: [ { custom_id: "ok", @@ -605,7 +598,7 @@ describe("OpenAI Batch instrumentation", () => { await completeOpenAIBatchTrace({ batch, - input: input(), + inputFile: input(), outputFile, webhook: { type: "batch.completed", @@ -634,7 +627,7 @@ describe("OpenAI Batch instrumentation", () => { await completeOpenAIBatchTrace({ batch, - input: input(), + inputFile: input(), outputFile, webhook: { type: "batch.completed", data: { id: "another_batch" } }, }); @@ -674,7 +667,7 @@ describe("OpenAI Batch instrumentation", () => { completed_at: completedAt, metadata: params.metadata, }, - input: [ + inputFile: [ { custom_id: "response_1", method: "POST", diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts index 14f855199..f6de6d12f 100644 --- a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts @@ -66,7 +66,7 @@ type BatchInputRecord = { spanData?: ReturnType; }; -type OpenAIBatchResultFile = +type OpenAIBatchFile = CompleteOpenAIBatchTraceArgs["outputFile"]; type PreparedBatch = { @@ -279,7 +279,7 @@ async function exportMinimalParent( } async function prepareCreateParams( - args: StartOpenAIBatchTraceArgs, + args: StartOpenAIBatchTraceArgs, parent: ReturnType, startTime: number, ): Promise { @@ -541,7 +541,7 @@ async function startBatchChild( } async function* jsonlRecords( - file: OpenAIBatchJSONL | OpenAIBatchResultFile, + file: OpenAIBatchJSONL | OpenAIBatchFile, onIssue: (error: Error) => void = () => {}, ): AsyncGenerator { const resolvedFile = await file; @@ -549,7 +549,12 @@ async function* jsonlRecords( return; } if (typeof resolvedFile === "string") { - for (const line of resolvedFile.split(/\r?\n/)) { + let offset = 0; + while (offset < resolvedFile.length) { + const newline = resolvedFile.indexOf("\n", offset); + const end = newline === -1 ? resolvedFile.length : newline; + const line = resolvedFile.slice(offset, end).replace(/\r$/, ""); + offset = newline === -1 ? resolvedFile.length : newline + 1; if (!line.trim()) { continue; } @@ -563,18 +568,7 @@ async function* jsonlRecords( return; } - const clone = read(resolvedFile, "clone"); - let response = resolvedFile; - if (typeof clone === "function") { - try { - response = Reflect.apply(clone, resolvedFile, []); - } catch (error) { - logBatchInstrumentationError("could not clone JSONL response", error); - onIssue(new Error("OpenAI Batch response could not be cloned")); - return; - } - } - const body = read(response, "body"); + const body = read(resolvedFile, "body"); const getReader = read(body, "getReader"); if (isObject(body) && typeof getReader === "function") { let reader: unknown; @@ -686,7 +680,7 @@ export const interceptOpenAIBatchTraceStart: Parameters< logBatchInstrumentationError("could not prepare batch", error); } - const batch = await Reflect.apply(target, thisArg, [ + const params = await Reflect.apply(target, thisArg, [ prepared ? { ...args[0], @@ -696,24 +690,12 @@ export const interceptOpenAIBatchTraceStart: Parameters< ]); if (prepared) { try { - const returnedContext = await batchContextFromBatch(batch); - if ( - !returnedContext || - batchContextPayload(returnedContext) !== - batchContextPayload(prepared.context) - ) { - logBatchInstrumentationError( - "did not start batch spans", - "returned batch context does not match the submitted batch", - ); - } else { - await startPreparedBatch(prepared.inputs, prepared.context); - } + await startPreparedBatch(prepared.inputs, prepared.context); } catch (error) { logBatchInstrumentationError("could not start batch spans", error); } } - return batch; + return params; }; function errorFromResult(result: BatchResultRecord): Error | undefined { @@ -781,7 +763,7 @@ async function completeBatchResult( } async function readBatchInputs( - file: OpenAIBatchResultFile, + file: OpenAIBatchFile, endpoint: string, ): Promise<{ inputDigest: string; @@ -919,7 +901,7 @@ async function completeResultFile({ }: { context: BatchContext; endTime: number; - file: OpenAIBatchResultFile; + file: OpenAIBatchFile; inputs: Map; issues: Error[]; seen: Set; @@ -1027,7 +1009,7 @@ async function completeBatch( webhook?.created_at, context.startTime, ); - const inputData = await readBatchInputs(args.input, context.endpoint); + const inputData = await readBatchInputs(args.inputFile, context.endpoint); const issues = [...inputData.issues]; if ( inputData.issues.length === 0 && diff --git a/js/src/instrumentation/plugins/openai-channels.ts b/js/src/instrumentation/plugins/openai-channels.ts index b966acd1c..445a65a68 100644 --- a/js/src/instrumentation/plugins/openai-channels.ts +++ b/js/src/instrumentation/plugins/openai-channels.ts @@ -19,6 +19,7 @@ import type { } from "../../vendor-sdk-types/openai"; import type { CompleteOpenAIBatchTraceArgs, + OpenAIBatchCreateParams, OpenAIBatchLike, StartOpenAIBatchTraceArgs, } from "../../openai-batch-types"; @@ -37,8 +38,8 @@ export const openAIChannels = defineChannels( "openai", { batchesStartTrace: channel< - [StartOpenAIBatchTraceArgs], - OpenAIBatchLike, + [StartOpenAIBatchTraceArgs], + OpenAIBatchCreateParams, OpenAIChannelExtras >({ channelName: "batches.start-trace", diff --git a/js/src/openai-batch-types.ts b/js/src/openai-batch-types.ts index 700ff8322..0781f9b01 100644 --- a/js/src/openai-batch-types.ts +++ b/js/src/openai-batch-types.ts @@ -3,11 +3,11 @@ export type OpenAIBatchJSONL = | Iterable | AsyncIterable; -type OpenAIBatchResultFileContent = OpenAIBatchJSONL | Response; +type OpenAIBatchFileContent = OpenAIBatchJSONL | Response; -type OpenAIBatchResultFile = - | OpenAIBatchResultFileContent - | PromiseLike; +type OpenAIBatchFile = + | OpenAIBatchFileContent + | PromiseLike; type SupportedOpenAIBatchEndpoint = "/v1/chat/completions" | "/v1/responses"; @@ -49,12 +49,6 @@ export interface OpenAIFileLike { [key: string]: unknown; } -export interface OpenAIBatchClientLike { - batches: { - create(params: OpenAIBatchCreateParams): PromiseLike; - }; -} - export interface OpenAIBatchWebhookEvent { type: | "batch.completed" @@ -66,8 +60,7 @@ export interface OpenAIBatchWebhookEvent { [key: string]: unknown; } -export interface StartOpenAIBatchTraceArgs { - client: OpenAIBatchClientLike; +export interface StartOpenAIBatchTraceArgs { inputFile: OpenAIFileLike; input: OpenAIBatchJSONL; params: OpenAIBatchCreateInputParams; @@ -75,8 +68,8 @@ export interface StartOpenAIBatchTraceArgs { export interface CompleteOpenAIBatchTraceArgs { batch: TBatch; - input: OpenAIBatchResultFile; - outputFile?: OpenAIBatchResultFile; - errorFile?: OpenAIBatchResultFile; + inputFile: OpenAIBatchFile; + outputFile?: OpenAIBatchFile; + errorFile?: OpenAIBatchFile; webhook?: OpenAIBatchWebhookEvent; } diff --git a/js/src/openai-batch.ts b/js/src/openai-batch.ts index db587ec7e..b0b4872eb 100644 --- a/js/src/openai-batch.ts +++ b/js/src/openai-batch.ts @@ -1,30 +1,29 @@ import { openAIChannels } from "./instrumentation/plugins/openai-channels"; import type { CompleteOpenAIBatchTraceArgs, + OpenAIBatchCreateParams, OpenAIBatchLike, StartOpenAIBatchTraceArgs, } from "./openai-batch-types"; /** - * Start a trace for a new OpenAI Batch, with pending spans for its requests. + * Start a trace for a new OpenAI Batch, with pending spans for its requests, + * and return the parameters to pass to `client.batches.create()`. * * Supported endpoints are `/v1/chat/completions` and `/v1/responses`. */ -export async function startOpenAIBatchTrace( - args: StartOpenAIBatchTraceArgs, -): Promise { - const batch = await openAIChannels.batchesStartTrace.invoke( - async (input) => - await input.client.batches.create({ - ...input.params, - input_file_id: input.inputFile.id, - }), +export async function startOpenAIBatchTrace( + args: StartOpenAIBatchTraceArgs, +): Promise { + return await openAIChannels.batchesStartTrace.invoke( + async (input) => ({ + ...input.params, + input_file_id: input.inputFile.id, + }), undefined, [args], {}, ); - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return batch as TBatch; } /** @@ -32,12 +31,13 @@ export async function startOpenAIBatchTrace( * * This helper performs no OpenAI API requests. Pass the Responses or promises * returned by `client.files.content()` for the input and result files, JSONL - * strings, or iterables of parsed records. + * strings, or iterables of parsed records. Supplied Response bodies are + * consumed directly. */ export async function completeOpenAIBatchTrace( args: CompleteOpenAIBatchTraceArgs, ): Promise { - const input = Promise.resolve(args.input); + const inputFile = Promise.resolve(args.inputFile); const outputFile = args.outputFile === undefined ? undefined @@ -48,14 +48,14 @@ export async function completeOpenAIBatchTrace( // Content requests are commonly started before this helper is called. Attach // rejection handlers immediately so early no-op paths and disabled // instrumentation cannot leave caller-supplied promises unhandled. - void input.catch(() => undefined); + void inputFile.catch(() => undefined); void outputFile?.catch(() => undefined); void errorFile?.catch(() => undefined); const batch = await openAIChannels.batchesCompleteTrace.invoke( async (input) => input.batch, undefined, - [{ ...args, input, outputFile, errorFile }], + [{ ...args, inputFile, outputFile, errorFile }], {}, ); // The completion channel preserves the exact batch object supplied by the From 2bd6ad167e8ff0b77b95bd2bda159e854074362c Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:06:37 +0000 Subject: [PATCH 5/6] refactor: clarify OpenAI batch terminal timestamps --- .../plugins/openai-batch-instrumentation.ts | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts index f6de6d12f..b4bc0cafc 100644 --- a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts @@ -959,23 +959,36 @@ function terminalEndTime( webhookCreatedAt: number | undefined, startTime: number, ): number { - const timestamp = - batch.status === "completed" - ? batch.completed_at - : batch.status === "failed" - ? batch.failed_at - : batch.status === "expired" - ? batch.expired_at - : batch.cancelled_at; - return typeof timestamp === "number" && + let timestamp: number | null | undefined; + switch (batch.status) { + case "completed": + timestamp = batch.completed_at; + break; + case "failed": + timestamp = batch.failed_at; + break; + case "expired": + timestamp = batch.expired_at; + break; + default: + timestamp = batch.cancelled_at; + } + + if ( + typeof timestamp === "number" && Number.isFinite(timestamp) && timestamp >= startTime - ? timestamp - : typeof webhookCreatedAt === "number" && - Number.isFinite(webhookCreatedAt) && - webhookCreatedAt >= startTime - ? webhookCreatedAt - : Math.max(getCurrentUnixTimestamp(), startTime); + ) { + return timestamp; + } + if ( + typeof webhookCreatedAt === "number" && + Number.isFinite(webhookCreatedAt) && + webhookCreatedAt >= startTime + ) { + return webhookCreatedAt; + } + return Math.max(getCurrentUnixTimestamp(), startTime); } async function completeBatch( From 11c6ef3847e85de165dc86d2499775a1667cfe83 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 20 Aug 2026 17:44:38 +0000 Subject: [PATCH 6/6] Update PR #2369 --- .../openai-v4-auto-hook.span-tree.json | 218 +- .../openai-v4-auto-hook.span-tree.txt | 2763 ++++++++------- .../openai-v4-latest-auto-hook.span-tree.json | 218 +- .../openai-v4-latest-auto-hook.span-tree.txt | 2763 ++++++++------- .../openai-v4-latest-wrapped.span-tree.json | 218 +- .../openai-v4-latest-wrapped.span-tree.txt | 2763 ++++++++------- .../openai-v4-wrapped.span-tree.json | 218 +- .../openai-v4-wrapped.span-tree.txt | 2763 ++++++++------- .../openai-v5-auto-hook.span-tree.json | 218 +- .../openai-v5-auto-hook.span-tree.txt | 2763 ++++++++------- .../openai-v5-latest-auto-hook.span-tree.json | 218 +- .../openai-v5-latest-auto-hook.span-tree.txt | 2763 ++++++++------- .../openai-v5-latest-wrapped.span-tree.json | 218 +- .../openai-v5-latest-wrapped.span-tree.txt | 2763 ++++++++------- .../openai-v5-wrapped.span-tree.json | 218 +- .../openai-v5-wrapped.span-tree.txt | 2763 ++++++++------- .../openai-v6-auto-hook.span-tree.json | 218 +- .../openai-v6-auto-hook.span-tree.txt | 3155 +++++++++-------- .../openai-v6-latest-auto-hook.span-tree.json | 218 +- .../openai-v6-latest-auto-hook.span-tree.txt | 3155 +++++++++-------- .../openai-v6-latest-wrapped.span-tree.json | 218 +- .../openai-v6-latest-wrapped.span-tree.txt | 3155 +++++++++-------- .../openai-v6-wrapped.span-tree.json | 218 +- .../openai-v6-wrapped.span-tree.txt | 3155 +++++++++-------- .../openai-instrumentation/scenario.impl.mjs | 98 +- js/src/exports.ts | 3 +- .../plugins/openai-batch-constants.ts | 1 - .../openai-batch-instrumentation.test.ts | 817 ++--- .../plugins/openai-batch-instrumentation.ts | 1038 ++---- .../plugins/openai-channels.ts | 26 +- .../instrumentation/plugins/openai-plugin.ts | 10 +- js/src/openai-batch-types.ts | 69 +- js/src/openai-batch.ts | 308 +- 33 files changed, 19688 insertions(+), 20022 deletions(-) delete mode 100644 js/src/instrumentation/plugins/openai-batch-constants.ts diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json index 1fc78fb5f..d6b8c55fe 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json @@ -768,113 +768,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1564,6 +1458,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt index 243c2ed0e..6c78bd0f3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt @@ -1,1404 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "4.104.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS.", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 16, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 62 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "4.104.0", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-chat-with-response-operation + │ } + │ ] │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-stream-operation + │ } + │ ] │ metadata: { - │ "operation": "stream", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 14, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - ├── openai-batch-operation - │ metadata: { - │ "operation": "batch", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS.", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS.", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL.", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - └── openai-responses-parse-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: "What is 20 + 4?" - output: [ - { - "content": [ - { - "annotations": [], - "logprobs": [], - "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", - "type": "output_text" - } - ], - "id": "", - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "background": false, - "billing": { - "payer": "developer" - }, - "completed_at": 0, - "created_at": 0, - "error": null, - "frequency_penalty": 0, - "id": "", - "incomplete_details": null, - "instructions": null, - "max_output_tokens": null, - "max_tool_calls": null, - "metadata": {}, - "model": "gpt-4o-mini-2024-07-18", - "moderation": null, - "object": "response", - "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", - "parallel_tool_calls": true, - "presence_penalty": 0, - "previous_response_id": null, - "prompt_cache_retention": "in_memory", - "provider": "openai", - "reasoning": { - "context": null, - "effort": null, - "summary": null - }, - "safety_identifier": null, - "service_tier": "default", - "status": "completed", - "store": true, - "temperature": 1, - "text": { - "format": { - "description": null, - "name": "NumberAnswer", - "schema": { - "additionalProperties": false, - "properties": { - "reasoning": { - "type": "string" - }, - "value": { - "type": "integer" - } - }, - "required": [ - "value", - "reasoning" - ], - "type": "object" - }, - "strict": true, - "type": "json_schema" - }, - "verbosity": "medium" - }, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "output_tokens": 0, - "output_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "total_tokens": 0 - }, - "web_search": { - "num_requests": 0 - } - }, - "tools": [], - "top_logprobs": 0, - "top_p": 1, - "truncation": "disabled", - "user": null - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 16, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 46, - "time_to_first_token": 0, - "tokens": 62 - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json index 362408cb3..95937b572 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json @@ -768,113 +768,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1564,6 +1458,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt index 32d22b074..87875d5b7 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt @@ -1,1404 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "4.104.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 18 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 22, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 68 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "4.104.0", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-chat-with-response-operation + │ } + │ ] │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-stream-operation + │ } + │ ] │ metadata: { - │ "operation": "stream", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 14, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - ├── openai-batch-operation - │ metadata: { - │ "operation": "batch", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM.", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 18 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - └── openai-responses-parse-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: "What is 20 + 4?" - output: [ - { - "content": [ - { - "annotations": [], - "logprobs": [], - "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", - "type": "output_text" - } - ], - "id": "", - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "background": false, - "billing": { - "payer": "developer" - }, - "completed_at": 0, - "created_at": 0, - "error": null, - "frequency_penalty": 0, - "id": "", - "incomplete_details": null, - "instructions": null, - "max_output_tokens": null, - "max_tool_calls": null, - "metadata": {}, - "model": "gpt-4o-mini-2024-07-18", - "moderation": null, - "object": "response", - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", - "parallel_tool_calls": true, - "presence_penalty": 0, - "previous_response_id": null, - "prompt_cache_retention": "in_memory", - "provider": "openai", - "reasoning": { - "context": null, - "effort": null, - "summary": null - }, - "safety_identifier": null, - "service_tier": "default", - "status": "completed", - "store": true, - "temperature": 1, - "text": { - "format": { - "description": null, - "name": "NumberAnswer", - "schema": { - "additionalProperties": false, - "properties": { - "reasoning": { - "type": "string" - }, - "value": { - "type": "integer" - } - }, - "required": [ - "value", - "reasoning" - ], - "type": "object" - }, - "strict": true, - "type": "json_schema" - }, - "verbosity": "medium" - }, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "output_tokens": 0, - "output_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "total_tokens": 0 - }, - "web_search": { - "num_requests": 0 - } - }, - "tools": [], - "top_logprobs": 0, - "top_p": 1, - "truncation": "disabled", - "user": null - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 22, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 46, - "time_to_first_token": 0, - "tokens": 68 - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json index 511c12796..8385ab9ee 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json @@ -760,113 +760,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1564,6 +1458,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt index a8aa0e11c..eba4aeaed 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt @@ -1,1404 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "4.104.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 18 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "parsed": { +│ "reasoning": "Adding 20 and 4 gives a total of 24.", +│ "value": 24 +│ }, +│ "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_parsed": { +│ "reasoning": "Adding 20 and 4 gives a total of 24.", +│ "value": 24 +│ }, +│ "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 22, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 68 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "4.104.0", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-chat-with-response-operation + │ } + │ ] │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-stream-operation + │ } + │ ] │ metadata: { - │ "operation": "stream", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "parsed": { - │ "answer": 4 - │ }, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "parsed": null, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - ├── openai-batch-operation - │ metadata: { - │ "operation": "batch", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM.", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 18 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - └── openai-responses-parse-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: "What is 20 + 4?" - output: [ - { - "content": [ - { - "annotations": [], - "logprobs": [], - "parsed": { - "reasoning": "Adding 20 and 4 gives a total of 24.", - "value": 24 - }, - "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", - "type": "output_text" - } - ], - "id": "", - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "background": false, - "billing": { - "payer": "developer" - }, - "completed_at": 0, - "created_at": 0, - "error": null, - "frequency_penalty": 0, - "id": "", - "incomplete_details": null, - "instructions": null, - "max_output_tokens": null, - "max_tool_calls": null, - "metadata": {}, - "model": "gpt-4o-mini-2024-07-18", - "moderation": null, - "object": "response", - "output_parsed": { - "reasoning": "Adding 20 and 4 gives a total of 24.", - "value": 24 - }, - "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", - "parallel_tool_calls": true, - "presence_penalty": 0, - "previous_response_id": null, - "prompt_cache_retention": "in_memory", - "provider": "openai", - "reasoning": { - "context": null, - "effort": null, - "summary": null - }, - "safety_identifier": null, - "service_tier": "default", - "status": "completed", - "store": true, - "temperature": 1, - "text": { - "format": { - "description": null, - "name": "NumberAnswer", - "schema": { - "additionalProperties": false, - "properties": { - "reasoning": { - "type": "string" - }, - "value": { - "type": "integer" - } - }, - "required": [ - "value", - "reasoning" - ], - "type": "object" - }, - "strict": true, - "type": "json_schema" - }, - "verbosity": "medium" - }, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "output_tokens": 0, - "output_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "total_tokens": 0 - }, - "web_search": { - "num_requests": 0 - } - }, - "tools": [], - "top_logprobs": 0, - "top_p": 1, - "truncation": "disabled", - "user": null - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 22, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 46, - "time_to_first_token": 0, - "tokens": 68 - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json index adf6d5a6b..1be92a331 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json @@ -760,113 +760,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1564,6 +1458,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt index 030c9e215..8050c979e 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt @@ -1,1404 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "4.104.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS.", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "parsed": { +│ "reasoning": "Twenty plus four equals twenty-four.", +│ "value": 24 +│ }, +│ "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_parsed": { +│ "reasoning": "Twenty plus four equals twenty-four.", +│ "value": 24 +│ }, +│ "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 16, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 62 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "4.104.0", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-chat-with-response-operation + │ } + │ ] │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-stream-operation + │ } + │ ] │ metadata: { - │ "operation": "stream", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "parsed": { - │ "answer": 4 - │ }, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "parsed": null, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - ├── openai-batch-operation - │ metadata: { - │ "operation": "batch", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS.", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS.", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL.", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - └── openai-responses-parse-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: "What is 20 + 4?" - output: [ - { - "content": [ - { - "annotations": [], - "logprobs": [], - "parsed": { - "reasoning": "Twenty plus four equals twenty-four.", - "value": 24 - }, - "text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", - "type": "output_text" - } - ], - "id": "", - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "background": false, - "billing": { - "payer": "developer" - }, - "completed_at": 0, - "created_at": 0, - "error": null, - "frequency_penalty": 0, - "id": "", - "incomplete_details": null, - "instructions": null, - "max_output_tokens": null, - "max_tool_calls": null, - "metadata": {}, - "model": "gpt-4o-mini-2024-07-18", - "moderation": null, - "object": "response", - "output_parsed": { - "reasoning": "Twenty plus four equals twenty-four.", - "value": 24 - }, - "output_text": "{\"reasoning\":\"Twenty plus four equals twenty-four.\",\"value\":24}", - "parallel_tool_calls": true, - "presence_penalty": 0, - "previous_response_id": null, - "prompt_cache_retention": "in_memory", - "provider": "openai", - "reasoning": { - "context": null, - "effort": null, - "summary": null - }, - "safety_identifier": null, - "service_tier": "default", - "status": "completed", - "store": true, - "temperature": 1, - "text": { - "format": { - "description": null, - "name": "NumberAnswer", - "schema": { - "additionalProperties": false, - "properties": { - "reasoning": { - "type": "string" - }, - "value": { - "type": "integer" - } - }, - "required": [ - "value", - "reasoning" - ], - "type": "object" - }, - "strict": true, - "type": "json_schema" - }, - "verbosity": "medium" - }, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "output_tokens": 0, - "output_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "total_tokens": 0 - }, - "web_search": { - "num_requests": 0 - } - }, - "tools": [], - "top_logprobs": 0, - "top_p": 1, - "truncation": "disabled", - "user": null - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 16, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 46, - "time_to_first_token": 0, - "tokens": 62 - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json index 8d614bc38..ed152f17e 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json @@ -768,113 +768,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1564,6 +1458,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt index 356a57f08..331725bf9 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt @@ -1,1404 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "5.11.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 18 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 26, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 72 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "5.11.0", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-chat-with-response-operation + │ } + │ ] │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-stream-operation + │ } + │ ] │ metadata: { - │ "operation": "stream", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 14, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - ├── openai-batch-operation - │ metadata: { - │ "operation": "batch", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM.", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 18 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - └── openai-responses-parse-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: "What is 20 + 4?" - output: [ - { - "content": [ - { - "annotations": [], - "logprobs": [], - "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", - "type": "output_text" - } - ], - "id": "", - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "background": false, - "billing": { - "payer": "developer" - }, - "completed_at": 0, - "created_at": 0, - "error": null, - "frequency_penalty": 0, - "id": "", - "incomplete_details": null, - "instructions": null, - "max_output_tokens": null, - "max_tool_calls": null, - "metadata": {}, - "model": "gpt-4o-mini-2024-07-18", - "moderation": null, - "object": "response", - "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", - "parallel_tool_calls": true, - "presence_penalty": 0, - "previous_response_id": null, - "prompt_cache_retention": "in_memory", - "provider": "openai", - "reasoning": { - "context": null, - "effort": null, - "summary": null - }, - "safety_identifier": null, - "service_tier": "default", - "status": "completed", - "store": true, - "temperature": 1, - "text": { - "format": { - "description": null, - "name": "NumberAnswer", - "schema": { - "additionalProperties": false, - "properties": { - "reasoning": { - "type": "string" - }, - "value": { - "type": "integer" - } - }, - "required": [ - "value", - "reasoning" - ], - "type": "object" - }, - "strict": true, - "type": "json_schema" - }, - "verbosity": "medium" - }, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "output_tokens": 0, - "output_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "total_tokens": 0 - }, - "web_search": { - "num_requests": 0 - } - }, - "tools": [], - "top_logprobs": 0, - "top_p": 1, - "truncation": "disabled", - "user": null - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 26, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 46, - "time_to_first_token": 0, - "tokens": 72 - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json index d059692a9..08ffe28d3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json @@ -768,113 +768,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1564,6 +1458,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt index 63909da75..dc15bc569 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt @@ -1,1404 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "5.23.2", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 35, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 81 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "5.23.2", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-chat-with-response-operation + │ } + │ ] │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-stream-operation + │ } + │ ] │ metadata: { - │ "operation": "stream", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 14, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - ├── openai-batch-operation - │ metadata: { - │ "operation": "batch", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - └── openai-responses-parse-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: "What is 20 + 4?" - output: [ - { - "content": [ - { - "annotations": [], - "logprobs": [], - "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", - "type": "output_text" - } - ], - "id": "", - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "background": false, - "billing": { - "payer": "developer" - }, - "completed_at": 0, - "created_at": 0, - "error": null, - "frequency_penalty": 0, - "id": "", - "incomplete_details": null, - "instructions": null, - "max_output_tokens": null, - "max_tool_calls": null, - "metadata": {}, - "model": "gpt-4o-mini-2024-07-18", - "moderation": null, - "object": "response", - "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", - "parallel_tool_calls": true, - "presence_penalty": 0, - "previous_response_id": null, - "prompt_cache_retention": "in_memory", - "provider": "openai", - "reasoning": { - "context": null, - "effort": null, - "summary": null - }, - "safety_identifier": null, - "service_tier": "default", - "status": "completed", - "store": true, - "temperature": 1, - "text": { - "format": { - "description": null, - "name": "NumberAnswer", - "schema": { - "additionalProperties": false, - "properties": { - "reasoning": { - "type": "string" - }, - "value": { - "type": "integer" - } - }, - "required": [ - "value", - "reasoning" - ], - "type": "object" - }, - "strict": true, - "type": "json_schema" - }, - "verbosity": "medium" - }, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "output_tokens": 0, - "output_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "total_tokens": 0 - }, - "web_search": { - "num_requests": 0 - } - }, - "tools": [], - "top_logprobs": 0, - "top_p": 1, - "truncation": "disabled", - "user": null - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 35, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 46, - "time_to_first_token": 0, - "tokens": 81 - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json index 230a290d6..ea5fc9fae 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json @@ -760,113 +760,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1564,6 +1458,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt index 1f2c5d78f..4a83f4dc2 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt @@ -1,1404 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "5.23.2", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "parsed": { +│ "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", +│ "value": 24 +│ }, +│ "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_parsed": { +│ "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", +│ "value": 24 +│ }, +│ "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 35, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 81 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "5.23.2", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-chat-with-response-operation + │ } + │ ] │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-stream-operation + │ } + │ ] │ metadata: { - │ "operation": "stream", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "parsed": { - │ "answer": 4 - │ }, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "parsed": null, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - ├── openai-batch-operation - │ metadata: { - │ "operation": "batch", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - └── openai-responses-parse-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: "What is 20 + 4?" - output: [ - { - "content": [ - { - "annotations": [], - "logprobs": [], - "parsed": { - "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", - "value": 24 - }, - "text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", - "type": "output_text" - } - ], - "id": "", - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "background": false, - "billing": { - "payer": "developer" - }, - "completed_at": 0, - "created_at": 0, - "error": null, - "frequency_penalty": 0, - "id": "", - "incomplete_details": null, - "instructions": null, - "max_output_tokens": null, - "max_tool_calls": null, - "metadata": {}, - "model": "gpt-4o-mini-2024-07-18", - "moderation": null, - "object": "response", - "output_parsed": { - "reasoning": "The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.", - "value": 24 - }, - "output_text": "{\"reasoning\":\"The sum of 20 and 4 is calculated by adding the two numbers together: 20 + 4 = 24.\",\"value\":24}", - "parallel_tool_calls": true, - "presence_penalty": 0, - "previous_response_id": null, - "prompt_cache_retention": "in_memory", - "provider": "openai", - "reasoning": { - "context": null, - "effort": null, - "summary": null - }, - "safety_identifier": null, - "service_tier": "default", - "status": "completed", - "store": true, - "temperature": 1, - "text": { - "format": { - "description": null, - "name": "NumberAnswer", - "schema": { - "additionalProperties": false, - "properties": { - "reasoning": { - "type": "string" - }, - "value": { - "type": "integer" - } - }, - "required": [ - "value", - "reasoning" - ], - "type": "object" - }, - "strict": true, - "type": "json_schema" - }, - "verbosity": "medium" - }, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "output_tokens": 0, - "output_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "total_tokens": 0 - }, - "web_search": { - "num_requests": 0 - } - }, - "tools": [], - "top_logprobs": 0, - "top_p": 1, - "truncation": "disabled", - "user": null - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 35, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 46, - "time_to_first_token": 0, - "tokens": 81 - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json index 7deccbd34..097c2c455 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json @@ -760,113 +760,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1564,6 +1458,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt index ba711bd95..54d9a2ee3 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt @@ -1,1404 +1,1409 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "5.11.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 18 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ └── openai-responses-parse-operation +│ metadata: { +│ "operation": "responses-parse", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.parse [llm] +│ input: "What is 20 + 4?" +│ output: [ +│ { +│ "content": [ +│ { +│ "annotations": [], +│ "logprobs": [], +│ "parsed": { +│ "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", +│ "value": 24 +│ }, +│ "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", +│ "type": "output_text" +│ } +│ ], +│ "id": "", +│ "role": "assistant", +│ "status": "completed", +│ "type": "message" +│ } +│ ] +│ metadata: { +│ "background": false, +│ "billing": { +│ "payer": "developer" +│ }, +│ "completed_at": 0, +│ "created_at": 0, +│ "error": null, +│ "frequency_penalty": 0, +│ "id": "", +│ "incomplete_details": null, +│ "instructions": null, +│ "max_output_tokens": null, +│ "max_tool_calls": null, +│ "metadata": {}, +│ "model": "gpt-4o-mini-2024-07-18", +│ "moderation": null, +│ "object": "response", +│ "output_parsed": { +│ "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", +│ "value": 24 +│ }, +│ "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", +│ "parallel_tool_calls": true, +│ "presence_penalty": 0, +│ "previous_response_id": null, +│ "prompt_cache_retention": "in_memory", +│ "provider": "openai", +│ "reasoning": { +│ "context": null, +│ "effort": null, +│ "summary": null +│ }, +│ "safety_identifier": null, +│ "service_tier": "default", +│ "status": "completed", +│ "store": true, +│ "temperature": 1, +│ "text": { +│ "format": { +│ "description": null, +│ "name": "NumberAnswer", +│ "schema": { +│ "additionalProperties": false, +│ "properties": { +│ "reasoning": { +│ "type": "string" +│ }, +│ "value": { +│ "type": "integer" +│ } +│ }, +│ "required": [ +│ "value", +│ "reasoning" +│ ], +│ "type": "object" +│ }, +│ "strict": true, +│ "type": "json_schema" +│ }, +│ "verbosity": "medium" +│ }, +│ "tool_choice": "auto", +│ "tool_usage": { +│ "image_gen": { +│ "input_tokens": 0, +│ "input_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "output_tokens": 0, +│ "output_tokens_details": { +│ "image_tokens": 0, +│ "text_tokens": 0 +│ }, +│ "total_tokens": 0 +│ }, +│ "web_search": { +│ "num_requests": 0 +│ } +│ }, +│ "tools": [], +│ "top_logprobs": 0, +│ "top_p": 1, +│ "truncation": "disabled", +│ "user": null +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 26, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 46, +│ "time_to_first_token": 0, +│ "tokens": 72 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "5.11.0", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - ├── openai-chat-with-response-operation + │ } + │ ] │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-stream-operation + │ } + │ ] │ metadata: { - │ "operation": "stream", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "parsed": { - │ "answer": 4 - │ }, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "parsed": null, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" - │ } - ├── openai-batch-operation - │ metadata: { - │ "operation": "batch", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM.", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 18 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - └── openai-responses-parse-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-parse", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.parse [llm] - input: "What is 20 + 4?" - output: [ - { - "content": [ - { - "annotations": [], - "logprobs": [], - "parsed": { - "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", - "value": 24 - }, - "text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", - "type": "output_text" - } - ], - "id": "", - "role": "assistant", - "status": "completed", - "type": "message" - } - ] - metadata: { - "background": false, - "billing": { - "payer": "developer" - }, - "completed_at": 0, - "created_at": 0, - "error": null, - "frequency_penalty": 0, - "id": "", - "incomplete_details": null, - "instructions": null, - "max_output_tokens": null, - "max_tool_calls": null, - "metadata": {}, - "model": "gpt-4o-mini-2024-07-18", - "moderation": null, - "object": "response", - "output_parsed": { - "reasoning": "20 plus 4 equals 24, as you simply add the two numbers together.", - "value": 24 - }, - "output_text": "{\"reasoning\":\"20 plus 4 equals 24, as you simply add the two numbers together.\",\"value\":24}", - "parallel_tool_calls": true, - "presence_penalty": 0, - "previous_response_id": null, - "prompt_cache_retention": "in_memory", - "provider": "openai", - "reasoning": { - "context": null, - "effort": null, - "summary": null - }, - "safety_identifier": null, - "service_tier": "default", - "status": "completed", - "store": true, - "temperature": 1, - "text": { - "format": { - "description": null, - "name": "NumberAnswer", - "schema": { - "additionalProperties": false, - "properties": { - "reasoning": { - "type": "string" - }, - "value": { - "type": "integer" - } - }, - "required": [ - "value", - "reasoning" - ], - "type": "object" - }, - "strict": true, - "type": "json_schema" - }, - "verbosity": "medium" - }, - "tool_choice": "auto", - "tool_usage": { - "image_gen": { - "input_tokens": 0, - "input_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "output_tokens": 0, - "output_tokens_details": { - "image_tokens": 0, - "text_tokens": 0 - }, - "total_tokens": 0 - }, - "web_search": { - "num_requests": 0 - } - }, - "tools": [], - "top_logprobs": 0, - "top_p": 1, - "truncation": "disabled", - "user": null - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 26, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 46, - "time_to_first_token": 0, - "tokens": 72 - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json index d49b81fe6..bf45c4c47 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json @@ -913,113 +913,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1784,6 +1678,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt index b4710287a..443e4a96e 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt @@ -1,1600 +1,1605 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "6.25.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-image-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-image-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Describe this image in three words or fewer.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "image_url": { +│ │ "url": { +│ │ "content_type": "image/png", +│ │ "filename": "image.png", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ } +│ │ }, +│ │ "type": "image_url" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Solid red.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 8516, +│ │ "time_to_first_token": 0, +│ │ "tokens": 8519 +│ │ } +│ ├── openai-chat-pdf-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-pdf-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Summarize this document in one short phrase.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "file": { +│ │ "file_data": { +│ │ "content_type": "application/pdf", +│ │ "filename": "document.pdf", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ }, +│ │ "filename": "document.pdf" +│ │ }, +│ │ "type": "file" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "length", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 236, +│ │ "time_to_first_token": 0, +│ │ "tokens": 260 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-parse-operation +│ │ metadata: { +│ │ "operation": "responses-parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.parse [llm] +│ │ input: "What is 20 + 4?" +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": null, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "description": null, +│ │ "name": "NumberAnswer", +│ │ "schema": { +│ │ "additionalProperties": false, +│ │ "properties": { +│ │ "reasoning": { +│ │ "type": "string" +│ │ }, +│ │ "value": { +│ │ "type": "integer" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "value", +│ │ "reasoning" +│ │ ], +│ │ "type": "object" +│ │ }, +│ │ "strict": true, +│ │ "type": "json_schema" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 17, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 46, +│ │ "time_to_first_token": 0, +│ │ "tokens": 63 +│ │ } +│ └── openai-responses-compact-operation +│ metadata: { +│ "operation": "responses-compact", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.compact [llm] +│ input: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "role": "user" +│ }, +│ { +│ "content": [ +│ { +│ "text": "Understood. I will keep answers concise.", +│ "type": "output_text" +│ } +│ ], +│ "role": "assistant" +│ } +│ ] +│ output: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "id": "", +│ "role": "user", +│ "status": "completed", +│ "type": "message" +│ }, +│ { +│ "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", +│ "id": "", +│ "type": "compaction" +│ } +│ ] +│ metadata: { +│ "created_at": 0, +│ "id": "", +│ "instructions": "Preserve only durable user preferences.", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "response.compaction", +│ "provider": "openai" +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 204, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 133, +│ "time_to_first_token": 0, +│ "tokens": 337 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "6.25.0", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-with-response-operation - │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-image-attachment-operation - │ metadata: { - │ "operation": "chat-image-attachment", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Describe this image in three words or fewer.", - │ "type": "text" - │ }, - │ { - │ "image_url": { - │ "url": { - │ "content_type": "image/png", - │ "filename": "image.png", - │ "key": "", - │ "type": "braintrust_attachment" - │ } - │ }, - │ "type": "image_url" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Solid red.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 8516, - │ "time_to_first_token": 0, - │ "tokens": 8519 - │ } - ├── openai-chat-pdf-attachment-operation - │ metadata: { - │ "operation": "chat-pdf-attachment", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Summarize this document in one short phrase.", - │ "type": "text" - │ }, - │ { - │ "file": { - │ "file_data": { - │ "content_type": "application/pdf", - │ "filename": "document.pdf", - │ "key": "", - │ "type": "braintrust_attachment" - │ }, - │ "filename": "document.pdf" - │ }, - │ "type": "file" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "length", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 24, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 236, - │ "time_to_first_token": 0, - │ "tokens": 260 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-operation - │ metadata: { - │ "operation": "stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 14, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-embeddings-operation + │ } + │ ] │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-batch-operation + │ } + │ ] │ metadata: { - │ "operation": "batch", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-parse-operation - │ metadata: { - │ "operation": "responses-parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.parse [llm] - │ input: "What is 20 + 4?" - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": null, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "description": null, - │ "name": "NumberAnswer", - │ "schema": { - │ "additionalProperties": false, - │ "properties": { - │ "reasoning": { - │ "type": "string" - │ }, - │ "value": { - │ "type": "integer" - │ } - │ }, - │ "required": [ - │ "value", - │ "reasoning" - │ ], - │ "type": "object" - │ }, - │ "strict": true, - │ "type": "json_schema" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 17, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 46, - │ "time_to_first_token": 0, - │ "tokens": 63 - │ } - └── openai-responses-compact-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-compact", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.compact [llm] - input: [ - { - "content": [ - { - "text": "I live in Paris and prefer concise answers.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "Understood. I will keep answers concise.", - "type": "output_text" - } - ], - "role": "assistant" - } - ] - output: [ - { - "content": [ - { - "text": "I live in Paris and prefer concise answers.", - "type": "input_text" - } - ], - "id": "", - "role": "user", - "status": "completed", - "type": "message" - }, - { - "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", - "id": "", - "type": "compaction" - } - ] - metadata: { - "created_at": 0, - "id": "", - "instructions": "Preserve only durable user preferences.", - "model": "gpt-4o-mini-2024-07-18", - "object": "response.compaction", - "provider": "openai" - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 204, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 133, - "time_to_first_token": 0, - "tokens": 337 - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json index 2bbb1383e..2dcad18c0 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json @@ -913,113 +913,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1784,6 +1678,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt index 254aa8fb6..b754f6b7c 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt @@ -1,1600 +1,1605 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "6.49.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-image-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-image-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Describe this image in three words or fewer.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "image_url": { +│ │ "url": { +│ │ "content_type": "image/png", +│ │ "filename": "image.png", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ } +│ │ }, +│ │ "type": "image_url" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Solid red.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 8516, +│ │ "time_to_first_token": 0, +│ │ "tokens": 8519 +│ │ } +│ ├── openai-chat-pdf-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-pdf-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Summarize this document in one short phrase.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "file": { +│ │ "file_data": { +│ │ "content_type": "application/pdf", +│ │ "filename": "document.pdf", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ }, +│ │ "filename": "document.pdf" +│ │ }, +│ │ "type": "file" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "length", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 236, +│ │ "time_to_first_token": 0, +│ │ "tokens": 260 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 14, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS.", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-parse-operation +│ │ metadata: { +│ │ "operation": "responses-parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.parse [llm] +│ │ input: "What is 20 + 4?" +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": null, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "description": null, +│ │ "name": "NumberAnswer", +│ │ "schema": { +│ │ "additionalProperties": false, +│ │ "properties": { +│ │ "reasoning": { +│ │ "type": "string" +│ │ }, +│ │ "value": { +│ │ "type": "integer" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "value", +│ │ "reasoning" +│ │ ], +│ │ "type": "object" +│ │ }, +│ │ "strict": true, +│ │ "type": "json_schema" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 46, +│ │ "time_to_first_token": 0, +│ │ "tokens": 70 +│ │ } +│ └── openai-responses-compact-operation +│ metadata: { +│ "operation": "responses-compact", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.compact [llm] +│ input: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "role": "user" +│ }, +│ { +│ "content": [ +│ { +│ "text": "Understood. I will keep answers concise.", +│ "type": "output_text" +│ } +│ ], +│ "role": "assistant" +│ } +│ ] +│ output: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "id": "", +│ "role": "user", +│ "status": "completed", +│ "type": "message" +│ }, +│ { +│ "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", +│ "id": "", +│ "type": "compaction" +│ } +│ ] +│ metadata: { +│ "created_at": 0, +│ "id": "", +│ "instructions": "Preserve only durable user preferences.", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "response.compaction", +│ "provider": "openai" +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 235, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 133, +│ "time_to_first_token": 0, +│ "tokens": 368 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "6.49.0", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-with-response-operation - │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-image-attachment-operation - │ metadata: { - │ "operation": "chat-image-attachment", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Describe this image in three words or fewer.", - │ "type": "text" - │ }, - │ { - │ "image_url": { - │ "url": { - │ "content_type": "image/png", - │ "filename": "image.png", - │ "key": "", - │ "type": "braintrust_attachment" - │ } - │ }, - │ "type": "image_url" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Solid red.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 8516, - │ "time_to_first_token": 0, - │ "tokens": 8519 - │ } - ├── openai-chat-pdf-attachment-operation - │ metadata: { - │ "operation": "chat-pdf-attachment", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Summarize this document in one short phrase.", - │ "type": "text" - │ }, - │ { - │ "file": { - │ "file_data": { - │ "content_type": "application/pdf", - │ "filename": "document.pdf", - │ "key": "", - │ "type": "braintrust_attachment" - │ }, - │ "filename": "document.pdf" - │ }, - │ "type": "file" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "length", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 24, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 236, - │ "time_to_first_token": 0, - │ "tokens": 260 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-operation - │ metadata: { - │ "operation": "stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 14, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-embeddings-operation + │ } + │ ] │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-batch-operation + │ } + │ ] │ metadata: { - │ "operation": "batch", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS.", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS.", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-parse-operation - │ metadata: { - │ "operation": "responses-parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.parse [llm] - │ input: "What is 20 + 4?" - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": null, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "description": null, - │ "name": "NumberAnswer", - │ "schema": { - │ "additionalProperties": false, - │ "properties": { - │ "reasoning": { - │ "type": "string" - │ }, - │ "value": { - │ "type": "integer" - │ } - │ }, - │ "required": [ - │ "value", - │ "reasoning" - │ ], - │ "type": "object" - │ }, - │ "strict": true, - │ "type": "json_schema" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 24, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 46, - │ "time_to_first_token": 0, - │ "tokens": 70 - │ } - └── openai-responses-compact-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-compact", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.compact [llm] - input: [ - { - "content": [ - { - "text": "I live in Paris and prefer concise answers.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "Understood. I will keep answers concise.", - "type": "output_text" - } - ], - "role": "assistant" - } - ] - output: [ - { - "content": [ - { - "text": "I live in Paris and prefer concise answers.", - "type": "input_text" - } - ], - "id": "", - "role": "user", - "status": "completed", - "type": "message" - }, - { - "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", - "id": "", - "type": "compaction" - } - ] - metadata: { - "created_at": 0, - "id": "", - "instructions": "Preserve only durable user preferences.", - "model": "gpt-4o-mini-2024-07-18", - "object": "response.compaction", - "provider": "openai" - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 235, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 133, - "time_to_first_token": 0, - "tokens": 368 - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json index 2f7daa87b..182d9da94 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json @@ -905,113 +905,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1784,6 +1678,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt index b70669bb1..c396bec2a 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt @@ -1,1600 +1,1605 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "6.49.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-image-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-image-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Describe this image in three words or fewer.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "image_url": { +│ │ "url": { +│ │ "content_type": "image/png", +│ │ "filename": "image.png", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ } +│ │ }, +│ │ "type": "image_url" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Solid red.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 8516, +│ │ "time_to_first_token": 0, +│ │ "tokens": 8519 +│ │ } +│ ├── openai-chat-pdf-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-pdf-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Summarize this document in one short phrase.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "file": { +│ │ "file_data": { +│ │ "content_type": "application/pdf", +│ │ "filename": "document.pdf", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ }, +│ │ "filename": "document.pdf" +│ │ }, +│ │ "type": "file" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "length", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 236, +│ │ "time_to_first_token": 0, +│ │ "tokens": 260 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS.", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS.", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-parse-operation +│ │ metadata: { +│ │ "operation": "responses-parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.parse [llm] +│ │ input: "What is 20 + 4?" +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "parsed": { +│ │ "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", +│ │ "value": 24 +│ │ }, +│ │ "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": null, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_parsed": { +│ │ "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", +│ │ "value": 24 +│ │ }, +│ │ "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "description": null, +│ │ "name": "NumberAnswer", +│ │ "schema": { +│ │ "additionalProperties": false, +│ │ "properties": { +│ │ "reasoning": { +│ │ "type": "string" +│ │ }, +│ │ "value": { +│ │ "type": "integer" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "value", +│ │ "reasoning" +│ │ ], +│ │ "type": "object" +│ │ }, +│ │ "strict": true, +│ │ "type": "json_schema" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 46, +│ │ "time_to_first_token": 0, +│ │ "tokens": 70 +│ │ } +│ └── openai-responses-compact-operation +│ metadata: { +│ "operation": "responses-compact", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.compact [llm] +│ input: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "role": "user" +│ }, +│ { +│ "content": [ +│ { +│ "text": "Understood. I will keep answers concise.", +│ "type": "output_text" +│ } +│ ], +│ "role": "assistant" +│ } +│ ] +│ output: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "id": "", +│ "role": "user", +│ "status": "completed", +│ "type": "message" +│ }, +│ { +│ "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", +│ "id": "", +│ "type": "compaction" +│ } +│ ] +│ metadata: { +│ "created_at": 0, +│ "id": "", +│ "instructions": "Preserve only durable user preferences.", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "response.compaction", +│ "provider": "openai" +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 235, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 133, +│ "time_to_first_token": 0, +│ "tokens": 368 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "6.49.0", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-with-response-operation - │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-image-attachment-operation - │ metadata: { - │ "operation": "chat-image-attachment", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Describe this image in three words or fewer.", - │ "type": "text" - │ }, - │ { - │ "image_url": { - │ "url": { - │ "content_type": "image/png", - │ "filename": "image.png", - │ "key": "", - │ "type": "braintrust_attachment" - │ } - │ }, - │ "type": "image_url" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Solid red.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 8516, - │ "time_to_first_token": 0, - │ "tokens": 8519 - │ } - ├── openai-chat-pdf-attachment-operation - │ metadata: { - │ "operation": "chat-pdf-attachment", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Summarize this document in one short phrase.", - │ "type": "text" - │ }, - │ { - │ "file": { - │ "file_data": { - │ "content_type": "application/pdf", - │ "filename": "document.pdf", - │ "key": "", - │ "type": "braintrust_attachment" - │ }, - │ "filename": "document.pdf" - │ }, - │ "type": "file" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "length", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 24, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 236, - │ "time_to_first_token": 0, - │ "tokens": 260 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-operation - │ metadata: { - │ "operation": "stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "parsed": { - │ "answer": 4 - │ }, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "parsed": null, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-embeddings-operation + │ } + │ ] │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-batch-operation + │ } + │ ] │ metadata: { - │ "operation": "batch", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS.", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS.", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-parse-operation - │ metadata: { - │ "operation": "responses-parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.parse [llm] - │ input: "What is 20 + 4?" - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "parsed": { - │ "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", - │ "value": 24 - │ }, - │ "text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": null, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_parsed": { - │ "reasoning": "Adding the two numbers together: 20 + 4 equals 24.", - │ "value": 24 - │ }, - │ "output_text": "{\"reasoning\":\"Adding the two numbers together: 20 + 4 equals 24.\",\"value\":24}", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "description": null, - │ "name": "NumberAnswer", - │ "schema": { - │ "additionalProperties": false, - │ "properties": { - │ "reasoning": { - │ "type": "string" - │ }, - │ "value": { - │ "type": "integer" - │ } - │ }, - │ "required": [ - │ "value", - │ "reasoning" - │ ], - │ "type": "object" - │ }, - │ "strict": true, - │ "type": "json_schema" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 24, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 46, - │ "time_to_first_token": 0, - │ "tokens": 70 - │ } - └── openai-responses-compact-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-compact", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.compact [llm] - input: [ - { - "content": [ - { - "text": "I live in Paris and prefer concise answers.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "Understood. I will keep answers concise.", - "type": "output_text" - } - ], - "role": "assistant" - } - ] - output: [ - { - "content": [ - { - "text": "I live in Paris and prefer concise answers.", - "type": "input_text" - } - ], - "id": "", - "role": "user", - "status": "completed", - "type": "message" - }, - { - "encrypted_content": "gAAAAABqfTl5Xd2K01sFMhngHE-y2hnvls368Fs_ERHUF-B0NWa_FFDqfovqsaohfgDnr8mG8DV5GdXBLmOSVvd65_JCu70TbkfaOuCnBfZqHgrDjvkGkHkKgB7k4FgrV9EKFJJN8a7Opfx-_5DLZZ6lAuIz1-Q-WB_fMH5uSeH-HRPmhBX0B5FXODmHpATJnsMNZ0ocbF7ruRO9IrurYgQ_G9w1rZCghmWgGGLHDCIznI5ZEr1ojEVjoMVFgxVxsKHaSFIKeKydgkHB3SNxJ2h7Ec3SN9pcKy6u0W0f7Jw9m1OnTaOg6IL_xHJmrFIqr5dzloqDQ33cXoQNn2cVlOROsRigKehYusyjhBGZ7JxMHYyTymEzSMZL5yfUmm2bIBMNB3VnYiC1nUgQ_KEuz7CQILdskccPsnlGaUjRet6IYHPqeXBq2l0tvzSSnaoFLTvTLq9nGmJy4OuYZTE4yJzS4KOi7Z9FPQ2BjZEa8IBw35bhMJuRLc0YjgJgFvr4c_wzes2LNXiv5GcZYlwvaGXCKddde1amextrC86IFENc5LCDGWjJ3GKom36aT6FabA0B_muWC9BPlVZVak9gIduilc8zoGdrsyttC6Y2tWksqYbdLPihoTw-b1iYaB0AcGSjtq_5i5RduYoBkhrXb1wN7NF9_Qn3JDexA9xcscUCIHgnKhmMiyw4n7bff_2874JqWmcmIpxnghUF3CGkOQ0AbjTSA9er6ByQYgVWCSIlMb6SfEl2ckv30DJ-hQ6tLbj7wlAgKpEFFCUby9uWRyCC_a0-B9IwYsqduM0PyzOXJlHnj1Wv5zXhtAIvnRnipep-Cgo4EF704C07CveKRtU8Ay-oRffUMYFux7vnKwarc80MjLV3uTY3DtzBwRKwTr0muFb2eSac4_eahQy08HMT3RVEyHNHTQ_0NMjSLJoNVCrFu82av2AL02SUB_PGKh61C5jl8IQK0teOqgACUcm4G1J3LmwoQ2AvMNbVD3cVapXzsN6Qp_xS7V7y1BTuUZ2euFQk9rjsPt_kpHEvosvw7FqDB4Tw1ICC1V8LVbN21XAQEGHavZLQ16QWEb2HCgrRjcP2APq68_L1XLUegZotdju7JoIDnPacn9iXITVkPRAFQ6Yrqq8AX0Ns88nED4rZXLFpnFN5xoq3J636f5FCnhWPQSc3CqfGX1IkAw_djWiEuhQqMhQsh-GX8ReXgRL1-KBM52_olNX4ZzDMYXF-eUla8ZWYC73IcjVxkNRjvJlg0Jnmq1HyQrfloPo1eGMXe7wdWCIz_1YmlAeW45VafQ6NrHKytdjcKU-04l3tFj2MmyNjHMHQ-WB7m0sDrW0cMFWeGB9nQWxcCg53k7VMj6OJ5ooM_xr7jZZl2zbQJxkGqRpavxBh_8C7mNcO1OJHkNuc5DaeGR_ov2hW5ZdvlPqpcU6waNzTU90gt2NgKPyHyoL-d3lqXWaZBfexd1zmrB_KwbxCNzmDWddOkpAo_ZNec6ugzh8-N5x6ciyuFqxcpeiM1NQL42lxHPmxeWBbUtIRjWPutAhk7x8evARSD2jWyiW99blzADqE7pzg1GpvePoj3Ip2FgrksWAdjEMkiLfhNx9B1w0049YfndHwwfUinAEBSWRMBlERvCLHzJTwrq8royEJSnIJ-vkSOxSa8B2vcKVDKPmSPZUBYst4kw_dQ6ZSXVibS1NtxRHy6xCFpvQX0So-uggliqM4RKn-dbX67Y5WpLCgxPKUmAU-RXWRkIFnMQsJD2lgo48PjW-eXq5x2T9WhVBwFwCTJVtScFY68SbnK4ZAHmLxf7SzG6vGeLFIxvyrgKHqHNphWdyO7C5VMEWqmsHPZIRrPjM85ECeiuske6ddQ03RWDD34dio9zwbClY6tQVXiHGE6mjPsdO7HexS4Zau8XO-y-AbURtQkq4IzEF8cqsoDXVP-xwrrHc_tJED-sD8m1zKStcZPPVJ1mkQ-AFh6d-C7d3P7PP_42CH6nTGg9C0CURFb3bsK_J-fmE6GWBf9dR0Kz9nbMYF6oDnaYCx8Kab4iZKEeR2J6f4C6HxIm6Nt8Qazjd8_YAL-I7NwCGagpKLsJ6wS-8Sp81NGNiIehAi64JnTKxLiOCyQOIPfC9wEJXT8KN1EXTPiKFtjqKuakpwJKuudMdbRnMYAOk4Vpr6XyQGLbeHG8c4jGYX01zhnI0yu1PaEWgi4wvOf5UnFVsa5UJ-GScUkmSEfrcY98OHL22cXvjejNwUkH_1Xzs8XgIMFpcQGMQ6f5RfFhdW_jsyc2iZMDPPnFyuTytPgZr-Zx55SRbQApi67fQCQ_ddvlh0gj1B7XZpROW2t-Px5NAZ4rhHYvfcORRqUthz-sB617EGbDbrDzrMQ9bA9je3SGQqCwGC9FnmK-yGpmcJc04qAvITOdzHVVEhraMNsV35LOgp3Wl1ALbO3KxIUmHE5xQs3mSkMXbQEJOQylelaq0o9BlPaeZXoeolBs8j-npHJ_aV92zcpOj1XcrB8imvUmk41XgPZ-6LdiYKs1kNSVEt9K__5LrsjDyiUAP-SP-PkuTig-oQbKO4sd79ztdxUuE9rTz3ScvJAJSBncH3Whg10X4YAOO7vfE9-mSgSxT4Z6HSI7qyWOwDm9aCRG_iwXGx7rQde67i93GvD-OrYlf43CEY2VEW-10vC0y0_Kdzkmox8-e2IcPXnxJQ-i0YKrJpnbMQ7TICay69GvytYBlccdV12uCvaWvyeLCH8MLTQLLE7-LDtXMK-jUfLq1iQq-R4kvlVYYs77yhIO57kzjPaD_3EBoOpQE4PZD53NYb3RZzOoe-KcrDoVJ8q2R79ekKOrby78307poTCWHfzZe0eExIWgmx606k_EEHJl4ELN0LOKEXP88BpRF5jSh70b324Zl4Hj3a7QRD0VWTTaq8G1vhoCvU31MQuHfGYJodWbR5ZSH68_XFzjtxPDoueN6ODB_XYqgLQkoaT1QAhZuqdB1SjvKJpALWp9Ccx0dLmddGZfXkS8t7smjFjCmuNbdSdzDJZ6oUbeZGetLwL22E4cyQL38a-N6hAnhv8__aSUQ6v9lRCSIZAFVJ-ApSo4OV72sAkp_WhzP4aA_e301UrbK6nZscenouwiPSuXObbyvG8yRvFswbX7_vkie5jJMD1mGujsq2Lnwtg9C502mL7hRWKZ6Pt4GSvmIUx1KsJdczXyYTfSWBYmT5kny08aiH1wInzGiz0-Oo7Mj-ptKOhTca3V5xuLNFLklC9Xy_ZXWM9nLmF-DWb1iTPf367-wzPIJbdS2iYakAsdvl43dxvapL8jfSTolZwGOaNSmbGSQOfAkQSI1-iiEmLnDUKAspXD92sk4J6BoF0CuFNXbhm3L9yTD5XT0=", - "id": "", - "type": "compaction" - } - ] - metadata: { - "created_at": 0, - "id": "", - "instructions": "Preserve only durable user preferences.", - "model": "gpt-4o-mini-2024-07-18", - "object": "response.compaction", - "provider": "openai" - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 235, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 133, - "time_to_first_token": 0, - "tokens": 368 - } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json index 5001d0fd8..586b97eaf 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json @@ -905,113 +905,7 @@ }, { "name": "openai-batch-operation", - "children": [ - { - "name": "openai.batch", - "type": "task", - "children": [ - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly ALPHA.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "ALPHA", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 9, - "tokens": 10 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly BRAVO.", - "role": "user" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "BRAVO", - "role": "assistant" - } - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 1, - "prompt_tokens": 8, - "tokens": 9 - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - }, - { - "name": "Chat Completion", - "type": "llm", - "children": [], - "input": [ - { - "content": "Reply with exactly CHARLIE.", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "error": "Batch fixture request failed", - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], - "metadata": { - "provider": "openai" - }, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - } - } - ], + "children": [], "metadata": { "operation": "batch", "testRunId": "" @@ -1784,6 +1678,116 @@ "caller_functionname": "runTracedScenario", "caller_lineno": 0 } + }, + { + "name": "openai.batch", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly ALPHA.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ALPHA", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_alpha", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 9, + "tokens": 10 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BRAVO.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "BRAVO", + "role": "assistant" + } + } + ], + "metadata": { + "custom_id": "batch_chat_bravo", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 8, + "tokens": 9 + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + }, + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ], + "metadata": { + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" + }, + "error": "Batch fixture request failed", + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } + } + ], + "metadata": { + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" + }, + "context": { + "caller_filename": "", + "caller_functionname": "", + "caller_lineno": 0 + } } ] } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt index 29e5058b6..1c6d3fe8d 100644 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt @@ -1,1600 +1,1605 @@ span_tree: -└── openai-instrumentation-root [task] +├── openai-instrumentation-root [task] +│ metadata: { +│ "openaiSdkVersion": "6.25.0", +│ "scenario": "openai-instrumentation", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runTracedScenario", +│ "caller_lineno": 0 +│ } +│ ├── openai-chat-operation +│ │ metadata: { +│ │ "operation": "chat", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly OK.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "OK.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-with-response-operation +│ │ metadata: { +│ │ "operation": "chat-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly FOUR.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Four.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 14 +│ │ } +│ ├── openai-chat-image-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-image-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Describe this image in three words or fewer.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "image_url": { +│ │ "url": { +│ │ "content_type": "image/png", +│ │ "filename": "image.png", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ } +│ │ }, +│ │ "type": "image_url" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "Solid red.", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 8516, +│ │ "time_to_first_token": 0, +│ │ "tokens": 8519 +│ │ } +│ ├── openai-chat-pdf-attachment-operation +│ │ metadata: { +│ │ "operation": "chat-pdf-attachment", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "text": "Summarize this document in one short phrase.", +│ │ "type": "text" +│ │ }, +│ │ { +│ │ "file": { +│ │ "file_data": { +│ │ "content_type": "application/pdf", +│ │ "filename": "document.pdf", +│ │ "key": "", +│ │ "type": "braintrust_attachment" +│ │ }, +│ │ "filename": "document.pdf" +│ │ }, +│ │ "type": "file" +│ │ } +│ │ ], +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "length", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 24, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 236, +│ │ "time_to_first_token": 0, +│ │ "tokens": 260 +│ │ } +│ ├── openai-chat-tool-operation +│ │ metadata: { +│ │ "operation": "chat-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": null, +│ │ "refusal": null, +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-operation +│ │ metadata: { +│ │ "operation": "stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 1, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 12, +│ │ "time_to_first_token": 0, +│ │ "tokens": 13 +│ │ } +│ ├── openai-stream-with-response-operation +│ │ metadata: { +│ │ "operation": "stream-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "STREAM-WITH-RESPONSE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 6, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 17, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-stream-tool-operation +│ │ metadata: { +│ │ "operation": "stream-tool", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "role": "assistant", +│ │ "tool_calls": [ +│ │ { +│ │ "function": { +│ │ "arguments": "{\"location\":\"Paris, France\"}", +│ │ "name": "get_weather" +│ │ }, +│ │ "id": "", +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 64, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "stream_options": { +│ │ "include_usage": true +│ │ }, +│ │ "temperature": 0, +│ │ "tool_choice": { +│ │ "function": { +│ │ "name": "get_weather" +│ │ }, +│ │ "type": "function" +│ │ }, +│ │ "tools": [ +│ │ { +│ │ "function": { +│ │ "description": "Get the weather for a location", +│ │ "name": "get_weather", +│ │ "parameters": { +│ │ "properties": { +│ │ "location": { +│ │ "description": "The location to get weather for", +│ │ "type": "string" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "location" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "function" +│ │ } +│ │ ] +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 7, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 77, +│ │ "time_to_first_token": 0, +│ │ "tokens": 84 +│ │ } +│ ├── openai-stream-fixture-operation +│ │ metadata: { +│ │ "operation": "stream-fixture", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with a refusal stream fixture.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": { +│ │ "content": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 78, +│ │ 79 +│ │ ], +│ │ "logprob": -0.1, +│ │ "token": "NO" +│ │ } +│ │ ] +│ │ }, +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE", +│ │ "top_logprobs": [ +│ │ { +│ │ "bytes": [ +│ │ 80, +│ │ 69 +│ │ ], +│ │ "logprob": -0.2, +│ │ "token": "PE" +│ │ } +│ │ ] +│ │ } +│ │ ] +│ │ }, +│ │ "message": { +│ │ "refusal": "NOPE", +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "logprobs": true, +│ │ "max_tokens": 12, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "stream": true, +│ │ "temperature": 0, +│ │ "top_logprobs": 2 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-parse-operation +│ │ metadata: { +│ │ "operation": "parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "What is 2 + 2?", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "annotations": [], +│ │ "content": "{\"answer\":4}", +│ │ "parsed": { +│ │ "answer": 4 +│ │ }, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "response_format": { +│ │ "json_schema": { +│ │ "name": "math_response", +│ │ "schema": { +│ │ "properties": { +│ │ "answer": { +│ │ "type": "number" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "answer" +│ │ ], +│ │ "type": "object" +│ │ } +│ │ }, +│ │ "type": "json_schema" +│ │ } +│ │ } +│ │ metrics: { +│ │ "completion_accepted_prediction_tokens": 0, +│ │ "completion_audio_tokens": 0, +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_rejected_prediction_tokens": 0, +│ │ "completion_tokens": 5, +│ │ "prompt_audio_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 45, +│ │ "time_to_first_token": 0, +│ │ "tokens": 50 +│ │ } +│ ├── openai-sync-stream-operation +│ │ metadata: { +│ │ "operation": "sync-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Chat Completion [llm] +│ │ input: [ +│ │ { +│ │ "content": "Reply with exactly SYNC STREAM.", +│ │ "role": "user" +│ │ } +│ │ ] +│ │ output: [ +│ │ { +│ │ "finish_reason": "stop", +│ │ "index": 0, +│ │ "logprobs": null, +│ │ "message": { +│ │ "content": "SYNC STREAM", +│ │ "parsed": null, +│ │ "refusal": null, +│ │ "role": "assistant" +│ │ } +│ │ } +│ │ ] +│ │ metadata: { +│ │ "max_tokens": 24, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "provider": "openai", +│ │ "temperature": 0 +│ │ } +│ │ metrics: { +│ │ "time_to_first_token": 0 +│ │ } +│ ├── openai-embeddings-operation +│ │ metadata: { +│ │ "operation": "embeddings", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Embedding [llm] +│ │ input: "Paris" +│ │ output: { +│ │ "embedding_length": 1536 +│ │ } +│ │ metadata: { +│ │ "model": "text-embedding-3-small", +│ │ "provider": "openai" +│ │ } +│ │ metrics: { +│ │ "prompt_tokens": 1, +│ │ "tokens": 1 +│ │ } +│ ├── openai-moderations-operation +│ │ metadata: { +│ │ "operation": "moderations", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── Moderation [llm] +│ │ input: "Hello from Braintrust." +│ │ output: [ +│ │ { +│ │ "categories": { +│ │ "harassment": false, +│ │ "harassment/threatening": false, +│ │ "hate": false, +│ │ "hate/threatening": false, +│ │ "illicit": false, +│ │ "illicit/violent": false, +│ │ "self-harm": false, +│ │ "self-harm/instructions": false, +│ │ "self-harm/intent": false, +│ │ "sexual": false, +│ │ "sexual/minors": false, +│ │ "violence": false, +│ │ "violence/graphic": false +│ │ }, +│ │ "category_applied_input_types": { +│ │ "harassment": [ +│ │ "text" +│ │ ], +│ │ "harassment/threatening": [ +│ │ "text" +│ │ ], +│ │ "hate": [ +│ │ "text" +│ │ ], +│ │ "hate/threatening": [ +│ │ "text" +│ │ ], +│ │ "illicit": [ +│ │ "text" +│ │ ], +│ │ "illicit/violent": [ +│ │ "text" +│ │ ], +│ │ "self-harm": [ +│ │ "text" +│ │ ], +│ │ "self-harm/instructions": [ +│ │ "text" +│ │ ], +│ │ "self-harm/intent": [ +│ │ "text" +│ │ ], +│ │ "sexual": [ +│ │ "text" +│ │ ], +│ │ "sexual/minors": [ +│ │ "text" +│ │ ], +│ │ "violence": [ +│ │ "text" +│ │ ], +│ │ "violence/graphic": [ +│ │ "text" +│ │ ] +│ │ }, +│ │ "category_scores": { +│ │ "harassment": 0.00001141223854434603, +│ │ "harassment/threatening": 0.0000012219076020491008, +│ │ "hate": 0.000003944258675190877, +│ │ "hate/threatening": 1.2878952156558146e-7, +│ │ "illicit": 0.000005738759521437678, +│ │ "illicit/violent": 0.000003822911232549001, +│ │ "self-harm": 0.000003120191139396651, +│ │ "self-harm/instructions": 6.240930669504435e-7, +│ │ "self-harm/intent": 0.0000022474089854291757, +│ │ "sexual": 0.000006814872211615988, +│ │ "sexual/minors": 5.255395707074638e-7, +│ │ "violence": 0.000014202364022997911, +│ │ "violence/graphic": 0.0000019525885208642222 +│ │ }, +│ │ "flagged": false +│ │ } +│ │ ] +│ │ metadata: { +│ │ "model": "omni-moderation-2024-09-26", +│ │ "provider": "openai" +│ │ } +│ ├── openai-batch-operation +│ │ metadata: { +│ │ "operation": "batch", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ ├── openai-responses-operation +│ │ metadata: { +│ │ "operation": "responses", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARIS." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARIS", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "PARIS", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-with-response-operation +│ │ metadata: { +│ │ "operation": "responses-with-response", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 2 + 2? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "4", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_text": "4", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-create-stream-operation +│ │ metadata: { +│ │ "operation": "responses-create-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly RESPONSE STREAM." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "RESPONSE STREAM", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "stream": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 4, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 17 +│ │ } +│ ├── openai-responses-stream-operation +│ │ metadata: { +│ │ "operation": "responses-stream", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "What is 6 x 6? Reply with just the number." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "36", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 2, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 21, +│ │ "time_to_first_token": 0, +│ │ "tokens": 23 +│ │ } +│ ├── openai-responses-stream-partial-operation +│ │ metadata: { +│ │ "operation": "responses-stream-partial", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.create [llm] +│ │ input: "Reply with exactly PARTIAL." +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "text": "PARTIAL", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": 24, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "type": "text" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 3, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 13, +│ │ "time_to_first_token": 0, +│ │ "tokens": 16 +│ │ } +│ ├── openai-responses-parse-operation +│ │ metadata: { +│ │ "operation": "responses-parse", +│ │ "testRunId": "" +│ │ } +│ │ context: { +│ │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ │ "caller_functionname": "runOperation", +│ │ "caller_lineno": 0 +│ │ } +│ │ └── openai.responses.parse [llm] +│ │ input: "What is 20 + 4?" +│ │ output: [ +│ │ { +│ │ "content": [ +│ │ { +│ │ "annotations": [], +│ │ "logprobs": [], +│ │ "parsed": { +│ │ "reasoning": "20 plus 4 equals 24.", +│ │ "value": 24 +│ │ }, +│ │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", +│ │ "type": "output_text" +│ │ } +│ │ ], +│ │ "id": "", +│ │ "role": "assistant", +│ │ "status": "completed", +│ │ "type": "message" +│ │ } +│ │ ] +│ │ metadata: { +│ │ "background": false, +│ │ "billing": { +│ │ "payer": "developer" +│ │ }, +│ │ "completed_at": 0, +│ │ "created_at": 0, +│ │ "error": null, +│ │ "frequency_penalty": 0, +│ │ "id": "", +│ │ "incomplete_details": null, +│ │ "instructions": null, +│ │ "max_output_tokens": null, +│ │ "max_tool_calls": null, +│ │ "metadata": {}, +│ │ "model": "gpt-4o-mini-2024-07-18", +│ │ "moderation": null, +│ │ "object": "response", +│ │ "output_parsed": { +│ │ "reasoning": "20 plus 4 equals 24.", +│ │ "value": 24 +│ │ }, +│ │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", +│ │ "parallel_tool_calls": true, +│ │ "presence_penalty": 0, +│ │ "previous_response_id": null, +│ │ "prompt_cache_retention": "in_memory", +│ │ "provider": "openai", +│ │ "reasoning": { +│ │ "context": null, +│ │ "effort": null, +│ │ "summary": null +│ │ }, +│ │ "safety_identifier": null, +│ │ "service_tier": "default", +│ │ "status": "completed", +│ │ "store": true, +│ │ "temperature": 1, +│ │ "text": { +│ │ "format": { +│ │ "description": null, +│ │ "name": "NumberAnswer", +│ │ "schema": { +│ │ "additionalProperties": false, +│ │ "properties": { +│ │ "reasoning": { +│ │ "type": "string" +│ │ }, +│ │ "value": { +│ │ "type": "integer" +│ │ } +│ │ }, +│ │ "required": [ +│ │ "value", +│ │ "reasoning" +│ │ ], +│ │ "type": "object" +│ │ }, +│ │ "strict": true, +│ │ "type": "json_schema" +│ │ }, +│ │ "verbosity": "medium" +│ │ }, +│ │ "tool_choice": "auto", +│ │ "tool_usage": { +│ │ "image_gen": { +│ │ "input_tokens": 0, +│ │ "input_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "output_tokens": 0, +│ │ "output_tokens_details": { +│ │ "image_tokens": 0, +│ │ "text_tokens": 0 +│ │ }, +│ │ "total_tokens": 0 +│ │ }, +│ │ "web_search": { +│ │ "num_requests": 0 +│ │ } +│ │ }, +│ │ "tools": [], +│ │ "top_logprobs": 0, +│ │ "top_p": 1, +│ │ "truncation": "disabled", +│ │ "user": null +│ │ } +│ │ metrics: { +│ │ "completion_reasoning_tokens": 0, +│ │ "completion_tokens": 17, +│ │ "prompt_cache_write_tokens": 0, +│ │ "prompt_cached_tokens": 0, +│ │ "prompt_tokens": 46, +│ │ "time_to_first_token": 0, +│ │ "tokens": 63 +│ │ } +│ └── openai-responses-compact-operation +│ metadata: { +│ "operation": "responses-compact", +│ "testRunId": "" +│ } +│ context: { +│ "caller_filename": "/e2e/helpers/provider-runtime.mjs", +│ "caller_functionname": "runOperation", +│ "caller_lineno": 0 +│ } +│ └── openai.responses.compact [llm] +│ input: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "role": "user" +│ }, +│ { +│ "content": [ +│ { +│ "text": "Understood. I will keep answers concise.", +│ "type": "output_text" +│ } +│ ], +│ "role": "assistant" +│ } +│ ] +│ output: [ +│ { +│ "content": [ +│ { +│ "text": "I live in Paris and prefer concise answers.", +│ "type": "input_text" +│ } +│ ], +│ "id": "", +│ "role": "user", +│ "status": "completed", +│ "type": "message" +│ }, +│ { +│ "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", +│ "id": "", +│ "type": "compaction" +│ } +│ ] +│ metadata: { +│ "created_at": 0, +│ "id": "", +│ "instructions": "Preserve only durable user preferences.", +│ "model": "gpt-4o-mini-2024-07-18", +│ "object": "response.compaction", +│ "provider": "openai" +│ } +│ metrics: { +│ "completion_reasoning_tokens": 0, +│ "completion_tokens": 204, +│ "prompt_cache_write_tokens": 0, +│ "prompt_cached_tokens": 0, +│ "prompt_tokens": 133, +│ "time_to_first_token": 0, +│ "tokens": 337 +│ } +└── openai.batch [task] metadata: { - "openaiSdkVersion": "6.25.0", - "scenario": "openai-instrumentation", - "testRunId": "" + "endpoint": "/v1/chat/completions", + "input_file_id": "file_batch_e2e_fixture", + "provider": "openai" } context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - ├── openai-chat-operation - │ metadata: { - │ "operation": "chat", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly OK.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "OK.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-with-response-operation - │ metadata: { - │ "operation": "chat-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly FOUR.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Four.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 14 - │ } - ├── openai-chat-image-attachment-operation - │ metadata: { - │ "operation": "chat-image-attachment", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Describe this image in three words or fewer.", - │ "type": "text" - │ }, - │ { - │ "image_url": { - │ "url": { - │ "content_type": "image/png", - │ "filename": "image.png", - │ "key": "", - │ "type": "braintrust_attachment" - │ } - │ }, - │ "type": "image_url" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "Solid red.", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 8516, - │ "time_to_first_token": 0, - │ "tokens": 8519 - │ } - ├── openai-chat-pdf-attachment-operation - │ metadata: { - │ "operation": "chat-pdf-attachment", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Summarize this document in one short phrase.", - │ "type": "text" - │ }, - │ { - │ "file": { - │ "file_data": { - │ "content_type": "application/pdf", - │ "filename": "document.pdf", - │ "key": "", - │ "type": "braintrust_attachment" - │ }, - │ "filename": "document.pdf" - │ }, - │ "type": "file" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "length", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "I can't access the content of the document to summarize it. If you can provide text or details from it, I'd be", - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 24, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 236, - │ "time_to_first_token": 0, - │ "tokens": 260 - │ } - ├── openai-chat-tool-operation - │ metadata: { - │ "operation": "chat-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": null, - │ "refusal": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-operation - │ metadata: { - │ "operation": "stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 1, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 12, - │ "time_to_first_token": 0, - │ "tokens": 13 - │ } - ├── openai-stream-with-response-operation - │ metadata: { - │ "operation": "stream-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "STREAM-WITH-RESPONSE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0 - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 6, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 17, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-stream-tool-operation - │ metadata: { - │ "operation": "stream-tool", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"location\":\"Paris, France\"}", - │ "name": "get_weather" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 64, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "stream_options": { - │ "include_usage": true - │ }, - │ "temperature": 0, - │ "tool_choice": { - │ "function": { - │ "name": "get_weather" - │ }, - │ "type": "function" - │ }, - │ "tools": [ - │ { - │ "function": { - │ "description": "Get the weather for a location", - │ "name": "get_weather", - │ "parameters": { - │ "properties": { - │ "location": { - │ "description": "The location to get weather for", - │ "type": "string" - │ } - │ }, - │ "required": [ - │ "location" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "function" - │ } - │ ] - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 7, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 77, - │ "time_to_first_token": 0, - │ "tokens": 84 - │ } - ├── openai-stream-fixture-operation - │ metadata: { - │ "operation": "stream-fixture", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with a refusal stream fixture.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": { - │ "content": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 78, - │ 79 - │ ], - │ "logprob": -0.1, - │ "token": "NO" - │ } - │ ] - │ }, - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE", - │ "top_logprobs": [ - │ { - │ "bytes": [ - │ 80, - │ 69 - │ ], - │ "logprob": -0.2, - │ "token": "PE" - │ } - │ ] - │ } - │ ] - │ }, - │ "message": { - │ "refusal": "NOPE", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "logprobs": true, - │ "max_tokens": 12, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "stream": true, - │ "temperature": 0, - │ "top_logprobs": 2 - │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-parse-operation - │ metadata: { - │ "operation": "parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "What is 2 + 2?", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "annotations": [], - │ "content": "{\"answer\":4}", - │ "parsed": { - │ "answer": 4 - │ }, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "response_format": { - │ "json_schema": { - │ "name": "math_response", - │ "schema": { - │ "properties": { - │ "answer": { - │ "type": "number" - │ } - │ }, - │ "required": [ - │ "answer" - │ ], - │ "type": "object" - │ } - │ }, - │ "type": "json_schema" - │ } - │ } - │ metrics: { - │ "completion_accepted_prediction_tokens": 0, - │ "completion_audio_tokens": 0, - │ "completion_reasoning_tokens": 0, - │ "completion_rejected_prediction_tokens": 0, - │ "completion_tokens": 5, - │ "prompt_audio_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 45, - │ "time_to_first_token": 0, - │ "tokens": 50 - │ } - ├── openai-sync-stream-operation - │ metadata: { - │ "operation": "sync-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly SYNC STREAM.", - │ "role": "user" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "logprobs": null, - │ "message": { - │ "content": "SYNC STREAM", - │ "parsed": null, - │ "refusal": null, - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "max_tokens": 24, - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai", - │ "temperature": 0 + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly ALPHA.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "ALPHA", + │ "role": "assistant" │ } - │ metrics: { - │ "time_to_first_token": 0 - │ } - ├── openai-embeddings-operation + │ } + │ ] │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 + │ "custom_id": "batch_chat_alpha", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ └── Embedding [llm] - │ input: "Paris" - │ output: { - │ "embedding_length": 1536 - │ } - │ metadata: { - │ "model": "text-embedding-3-small", - │ "provider": "openai" - │ } - │ metrics: { - │ "prompt_tokens": 1, - │ "tokens": 1 - │ } - ├── openai-moderations-operation - │ metadata: { - │ "operation": "moderations", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 9, + │ "tokens": 10 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── Moderation [llm] - │ input: "Hello from Braintrust." - │ output: [ - │ { - │ "categories": { - │ "harassment": false, - │ "harassment/threatening": false, - │ "hate": false, - │ "hate/threatening": false, - │ "illicit": false, - │ "illicit/violent": false, - │ "self-harm": false, - │ "self-harm/instructions": false, - │ "self-harm/intent": false, - │ "sexual": false, - │ "sexual/minors": false, - │ "violence": false, - │ "violence/graphic": false - │ }, - │ "category_applied_input_types": { - │ "harassment": [ - │ "text" - │ ], - │ "harassment/threatening": [ - │ "text" - │ ], - │ "hate": [ - │ "text" - │ ], - │ "hate/threatening": [ - │ "text" - │ ], - │ "illicit": [ - │ "text" - │ ], - │ "illicit/violent": [ - │ "text" - │ ], - │ "self-harm": [ - │ "text" - │ ], - │ "self-harm/instructions": [ - │ "text" - │ ], - │ "self-harm/intent": [ - │ "text" - │ ], - │ "sexual": [ - │ "text" - │ ], - │ "sexual/minors": [ - │ "text" - │ ], - │ "violence": [ - │ "text" - │ ], - │ "violence/graphic": [ - │ "text" - │ ] - │ }, - │ "category_scores": { - │ "harassment": 0.00001141223854434603, - │ "harassment/threatening": 0.0000012219076020491008, - │ "hate": 0.000003944258675190877, - │ "hate/threatening": 1.2878952156558146e-7, - │ "illicit": 0.000005738759521437678, - │ "illicit/violent": 0.000003822911232549001, - │ "self-harm": 0.000003120191139396651, - │ "self-harm/instructions": 6.240930669504435e-7, - │ "self-harm/intent": 0.0000022474089854291757, - │ "sexual": 0.000006814872211615988, - │ "sexual/minors": 5.255395707074638e-7, - │ "violence": 0.000014202364022997911, - │ "violence/graphic": 0.0000019525885208642222 - │ }, - │ "flagged": false - │ } - │ ] - │ metadata: { - │ "model": "omni-moderation-2024-09-26", - │ "provider": "openai" + ├── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BRAVO.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "BRAVO", + │ "role": "assistant" │ } - ├── openai-batch-operation + │ } + │ ] │ metadata: { - │ "operation": "batch", - │ "testRunId": "" + │ "custom_id": "batch_chat_bravo", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai" │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.batch [task] - │ metadata: { - │ "provider": "openai" - │ } - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly ALPHA.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "ALPHA", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 9, - │ │ "tokens": 10 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ ├── Chat Completion [llm] - │ │ input: [ - │ │ { - │ │ "content": "Reply with exactly BRAVO.", - │ │ "role": "user" - │ │ } - │ │ ] - │ │ output: [ - │ │ { - │ │ "finish_reason": "stop", - │ │ "index": 0, - │ │ "message": { - │ │ "content": "BRAVO", - │ │ "role": "assistant" - │ │ } - │ │ } - │ │ ] - │ │ metadata: { - │ │ "model": "gpt-4o-mini-2024-07-18", - │ │ "provider": "openai" - │ │ } - │ │ metrics: { - │ │ "completion_tokens": 1, - │ │ "prompt_tokens": 8, - │ │ "tokens": 9 - │ │ } - │ │ context: { - │ │ "caller_filename": "", - │ │ "caller_functionname": "", - │ │ "caller_lineno": 0 - │ │ } - │ └── Chat Completion [llm] - │ input: [ - │ { - │ "content": "Reply with exactly CHARLIE.", - │ "role": "user" - │ } - │ ] - │ metadata: { - │ "model": "gpt-4o-mini-2024-07-18", - │ "provider": "openai" - │ } - │ error: "Batch fixture request failed" - │ context: { - │ "caller_filename": "", - │ "caller_functionname": "", - │ "caller_lineno": 0 - │ } - ├── openai-responses-operation - │ metadata: { - │ "operation": "responses", - │ "testRunId": "" + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 8, + │ "tokens": 9 │ } │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", + │ "caller_filename": "", + │ "caller_functionname": "", │ "caller_lineno": 0 │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARIS." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARIS", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "PARIS", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-with-response-operation - │ metadata: { - │ "operation": "responses-with-response", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 2 + 2? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "4", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_text": "4", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-create-stream-operation - │ metadata: { - │ "operation": "responses-create-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly RESPONSE STREAM." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "RESPONSE STREAM", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "stream": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 4, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 17 - │ } - ├── openai-responses-stream-operation - │ metadata: { - │ "operation": "responses-stream", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "What is 6 x 6? Reply with just the number." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "36", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 2, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 21, - │ "time_to_first_token": 0, - │ "tokens": 23 - │ } - ├── openai-responses-stream-partial-operation - │ metadata: { - │ "operation": "responses-stream-partial", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.create [llm] - │ input: "Reply with exactly PARTIAL." - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "text": "PARTIAL", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": 24, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "type": "text" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 3, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 13, - │ "time_to_first_token": 0, - │ "tokens": 16 - │ } - ├── openai-responses-parse-operation - │ metadata: { - │ "operation": "responses-parse", - │ "testRunId": "" - │ } - │ context: { - │ "caller_filename": "/e2e/helpers/provider-runtime.mjs", - │ "caller_functionname": "runOperation", - │ "caller_lineno": 0 - │ } - │ └── openai.responses.parse [llm] - │ input: "What is 20 + 4?" - │ output: [ - │ { - │ "content": [ - │ { - │ "annotations": [], - │ "logprobs": [], - │ "parsed": { - │ "reasoning": "20 plus 4 equals 24.", - │ "value": 24 - │ }, - │ "text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", - │ "type": "output_text" - │ } - │ ], - │ "id": "", - │ "role": "assistant", - │ "status": "completed", - │ "type": "message" - │ } - │ ] - │ metadata: { - │ "background": false, - │ "billing": { - │ "payer": "developer" - │ }, - │ "completed_at": 0, - │ "created_at": 0, - │ "error": null, - │ "frequency_penalty": 0, - │ "id": "", - │ "incomplete_details": null, - │ "instructions": null, - │ "max_output_tokens": null, - │ "max_tool_calls": null, - │ "metadata": {}, - │ "model": "gpt-4o-mini-2024-07-18", - │ "moderation": null, - │ "object": "response", - │ "output_parsed": { - │ "reasoning": "20 plus 4 equals 24.", - │ "value": 24 - │ }, - │ "output_text": "{\"reasoning\":\"20 plus 4 equals 24.\",\"value\":24}", - │ "parallel_tool_calls": true, - │ "presence_penalty": 0, - │ "previous_response_id": null, - │ "prompt_cache_retention": "in_memory", - │ "provider": "openai", - │ "reasoning": { - │ "context": null, - │ "effort": null, - │ "summary": null - │ }, - │ "safety_identifier": null, - │ "service_tier": "default", - │ "status": "completed", - │ "store": true, - │ "temperature": 1, - │ "text": { - │ "format": { - │ "description": null, - │ "name": "NumberAnswer", - │ "schema": { - │ "additionalProperties": false, - │ "properties": { - │ "reasoning": { - │ "type": "string" - │ }, - │ "value": { - │ "type": "integer" - │ } - │ }, - │ "required": [ - │ "value", - │ "reasoning" - │ ], - │ "type": "object" - │ }, - │ "strict": true, - │ "type": "json_schema" - │ }, - │ "verbosity": "medium" - │ }, - │ "tool_choice": "auto", - │ "tool_usage": { - │ "image_gen": { - │ "input_tokens": 0, - │ "input_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "output_tokens": 0, - │ "output_tokens_details": { - │ "image_tokens": 0, - │ "text_tokens": 0 - │ }, - │ "total_tokens": 0 - │ }, - │ "web_search": { - │ "num_requests": 0 - │ } - │ }, - │ "tools": [], - │ "top_logprobs": 0, - │ "top_p": 1, - │ "truncation": "disabled", - │ "user": null - │ } - │ metrics: { - │ "completion_reasoning_tokens": 0, - │ "completion_tokens": 17, - │ "prompt_cache_write_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 46, - │ "time_to_first_token": 0, - │ "tokens": 63 - │ } - └── openai-responses-compact-operation + └── Chat Completion [llm] + input: [ + { + "content": "Reply with exactly CHARLIE.", + "role": "user" + } + ] metadata: { - "operation": "responses-compact", - "testRunId": "" + "custom_id": "batch_chat_charlie", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai" } + error: "Batch fixture request failed" context: { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", + "caller_filename": "", + "caller_functionname": "", "caller_lineno": 0 } - └── openai.responses.compact [llm] - input: [ - { - "content": [ - { - "text": "I live in Paris and prefer concise answers.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "Understood. I will keep answers concise.", - "type": "output_text" - } - ], - "role": "assistant" - } - ] - output: [ - { - "content": [ - { - "text": "I live in Paris and prefer concise answers.", - "type": "input_text" - } - ], - "id": "", - "role": "user", - "status": "completed", - "type": "message" - }, - { - "encrypted_content": "gAAAAABqfTlfO3le3ofWfPpoD_GoTqr0pqQtjefKMUqYXUJviCcZizEgU4Li-xOqvxe3QLlUR0n4dZaygcnBrhork5XCaGiYnN7kFF5ZK82uukJLfxecktwnavtlIcIbNQ_96cfqaQcHymGzbC95xk3AhHeHCmCQdaDO3IcNv21CnrogYCLRRH2hvRDYEqqwoJlJlopLgqCAEDs-5fNmrgRw_oHqhqGWg5HniRwFsleHlOYqP1lvDFSrXa30kmkgpWwqzYwmpZJcRwhteo73s6V7utVNhjykOshNqRMObjaJ06tPR5XtnOT61l9jwGvJ-TuTa6CtHjBB9w43Gt4VWGjzBERkqL2ieiCcx45gEwlgZFFBaXw1TfiFUPZeroFqLKzvkbRexLlDDP_xwRozID4zy-_-LEb95E4D7ugYMaUOax-VkBQa8YErdSJl98SxAsEzZzJypf5oBtt7g5rA5CriX43Cyks_pyIBb8HQFE_Q0Uiq26vcQVADFhHNfslFcUpC_vlCcKgYGGvzKvnUceNT2y7xgWZl0JMkj3w6Vjw1BygaRJFk5t0_GNHAdlyKww7p_GaGUR3ThET9Cio4CdwrCOHFOuGeqxtQBHN58EKouh0_XrVn-nd5KV4Bds5ujbPE_qneyCW7vA-0rDkDjbV4ZQ98qcNncUisf5Jg7b2QMS6VV02vKulsK1IIPefqsWRssBj5isFFidOukSCwdGyzLoM6wfa2XFZZFmm6ysDrTJoWB2xqxP7WHDuhzUDbPCYCqOJZNgPkR5TnlKeBapEWWUoLEPFR2x4yEx7exlP3GhmcTQ2ums00MTPERdxFH0QoC1gwui1Yk9qqmQIKIL45XNxsE9UxxuMynXZz2ERRqXQfYosI9PLgdCHLN6JZDYMj0D5PVeGEdL9xQVmBx_aw6uIMUrRhUOO1nPbZIiuLhT6eDr4u0x9qOREzzhjnu5sTFckS6eBXLO3Kdx8X4ygj-JSbexqBIYvB2mLduP3BIqCC9ZTbHiwVVxsdiuONpKFE8nYX93qvb-cNDQjUpgtxsc2Xq928TovxibcTT8jEQyhX1Fuzlw-SkyFMCJ8V4WMUtu8-QeVgJIMycaCDlXyL-fmkImMiHEJe9bak_g3h8-g4lh8FtHr_c8vvZK-vV4LGksWPSLjZarR2nFLVmUchDMtyqb0-y1JT0tftZMu3KbuRSsc68j7hfHFWVib8x4H3ZHSuzSih68QfJixRbGpjeSw6g6IgGx86ShklyNO5ElKQvbYmGHON97rd1oYvkYMxutIfgxfzo2JLlKETHh7_TGnhHSR1jaR1lrM6Jkg9CORZPqjfOZpRIDY4tuJqStaDBlJogdajPL7kWdmue7vB_7b2x-tJkCgpMExeV8oo3VZOxpdjDIzyJiuayZmUGBKRUV8KGLyj5j-kBwO6slyP07szZUpa_ZtScHd_5to97mIMp2Ae95qCOwuAQC63dyIBV7peyJ-jgE1MXUR1rD0Ws18raLIt-xAY3lSzsm2-YVfNVzCGwukQMKhAFt8yEjvTKc_tqQMst4M79H6te5ePGwteFwh1vnUXb3NZCmdWxvAe0Ln3bOnSV38yE8SS9MwP_G792C1ziW3yX_egpFHBolLV8GtZ8HRYfj_5x-1Ho8AtVx6IfR7BObuLn6PzHhLQLI0X2ClR_qJPgj0i7DvFzASO4odGjNa-FKh21WSoFs3tInpKibDSmMUlm-WNsgKpcJ_n8WB8ebEDFXcXKqg41Gq-ZdLC4q7vC5gLC-lgwaXlVuziGC3Q8gEQZ-AKhX2QV18Ct6Tlhdoyo6wZOlT-c3oR-L5z3wYM81hY1ubydLVvsIC7vMFAxEUJ-bWEkUnMZolUt0_tVU7QdN9qb2kdLR9Rbx38yBMuXZDYeferYM0031oGWGuLIXMO5T_aG8t9nBsiAF_8mDd6-zY4CKZYjS36JDtNkITvTAbv9tfSkrLo1XndFIjrx0-tAhk2dDUGuFaeP4ORb1ZIy4Lrq3Y13Z9GuawvvZU02Ikd_1_jLhjRhuqNP0fxmFt33GYaZ5sOSuWgRkOiy-noW_K0YExtRPpHAaJ8v5UNjQ3NicStMreICBvtzkgR4Nphq-MGCX2_YyLJB_HX_psCiR9qmBbcbp1DTYU0XO1QvV2RUmCXZN6kyqdRB8DfcBujxc8x-zt0BaXHXi_0_Qtdppf_nhUYEraUT_Q6917CR784w-cjXpEGKjgby3B4zCN-yF4Y4vDzfo7eF-L2IWbtviMoesRrnMalD36d-11hhFclmv4V8j8raEA2e1Hk6fO0X4OC055Z-58iujXWZNLtC68nzWnTGuKba5-9z1wtOrraXLVCPzaFM8DebeVx7OxeiejpLFzemtdlJVqArppTWEV-3Cg0UsRky2QlywjHOQ3G6aglYNQuw-omr0UR3IOyi7tw4diXsOwlymC1WRCs-gZLmU6mfzh_0q7YvAoAbHrGpEFHV2wpBdPYuF9qCo18y-LCtOGCYV0_W0ipFMtITmBF4_3AnBx6yeeuTkIiNHlt5B3RmbvR3-a-HTdGtPABM6yr93bnR0cUghCfH_sUm-w3uPYA0vA4P4bobmPk022CJzLwmMbiMLjOQK3r0STD_cdk1sBgNkT5wewnx8QwkryLvhyydxzb8mN-BneAHFVE8UT5PJQWC-jBSleZ5Z6cVXGDY5jwpdFLHj72auO7p916lkQn2-XUDHvkDu11MggSOhJsvH23AVuadP4aCbG5O68RbNIimrFOgboms1lSs9I0cXRbRWcai7ixPlRo6-Asi8gGq-l1qQQEluNy_uRZXlHPyvp0mNFS-1EvTqnDbXYEHxcgKdbuuBjaxIt-JTagHnXc2u14v5JCyXuZNwY9U1vX1DYKeacPTdWhMgI7jiwig2ljJo7aT6MesviJ3z7NY2zTiOMaDbMO2CcD3gp--WsA4c4lpx6gqH487yhyHMD3flIOZMKAjsqCZXFmhKR8pjCY_NzSkKBCoHpiL5oGHNP5iI9IcOVPivOKhdupAJIj4R5xGX3_P1WP2n1YSUN5Bpq04F0pmOBYqS5X5mFWcgechLH-evL-m6UMTX36105AVlL8SBCsGOMuXpWPiykg99oHSOH51oFRhwt3j3pBVs1SjMHW4sfZUZ2eeOxGswA7WmvUQ6LCLqc52n06J1JxYf1Byi0ggXZBLCMSv_9GU0ec6WERXHvCkotYa5HbMMRptnfGFDIWtVfgRg==", - "id": "", - "type": "compaction" - } - ] - metadata: { - "created_at": 0, - "id": "", - "instructions": "Preserve only durable user preferences.", - "model": "gpt-4o-mini-2024-07-18", - "object": "response.compaction", - "provider": "openai" - } - metrics: { - "completion_reasoning_tokens": 0, - "completion_tokens": 204, - "prompt_cache_write_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 133, - "time_to_first_token": 0, - "tokens": 337 - } diff --git a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs index 9dc04604a..48199411e 100644 --- a/e2e/scenarios/openai-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/openai-instrumentation/scenario.impl.mjs @@ -7,7 +7,11 @@ import { runOperation, runTracedScenario, } from "../../helpers/provider-runtime.mjs"; -import { completeOpenAIBatchTrace, startOpenAIBatchTrace } from "braintrust"; +import { + completeOpenAIBatchTrace, + openaiBatchesRetrieveTraced, + openaiFilesCreateTraced, +} from "braintrust"; const OPENAI_MODEL = "gpt-4o-mini-2024-07-18"; const EMBEDDING_MODEL = "text-embedding-3-small"; @@ -106,10 +110,57 @@ function createMockStreamingClient(options) { } function createMockBatchClient(options) { + const batchCreatedAt = Date.now() / 1000; const baseClient = new options.OpenAI({ apiKey: process.env.OPENAI_API_KEY ?? "test-openai-key", baseURL: "https://example.test/v1", - fetch: async (_url, init) => { + fetch: async (url, init) => { + const pathname = new URL(String(url)).pathname; + if (pathname.endsWith("/files")) { + return new Response( + JSON.stringify({ + id: "file_batch_e2e_fixture", + object: "file", + bytes: 1024, + created_at: batchCreatedAt, + filename: "batch.jsonl", + purpose: "batch", + status: "processed", + }), + { + headers: { + "content-type": "application/json", + "x-request-id": "req_file_batch_e2e_fixture", + }, + status: 200, + }, + ); + } + + if (pathname.endsWith("/batches/batch_e2e_fixture")) { + return new Response( + JSON.stringify({ + id: "batch_e2e_fixture", + object: "batch", + endpoint: "/v1/chat/completions", + input_file_id: "file_batch_e2e_fixture", + completion_window: "24h", + status: "completed", + created_at: batchCreatedAt, + in_progress_at: batchCreatedAt + 1, + completed_at: batchCreatedAt + 2, + request_counts: { completed: 2, failed: 1, total: 3 }, + }), + { + headers: { + "content-type": "application/json", + "x-request-id": "req_batch_e2e_fixture", + }, + status: 200, + }, + ); + } + const params = JSON.parse(String(init?.body)); return new Response( JSON.stringify({ @@ -650,24 +701,20 @@ export async function runOpenAIInstrumentationScenario(options) { }), ) .join("\n"); - const batchParams = await startOpenAIBatchTrace({ - inputFile: { id: "file_batch_e2e_fixture" }, - input, - params: { - completion_window: "24h", - endpoint: "/v1/chat/completions", - }, + const inputFile = await openaiFilesCreateTraced( + batchFixtureClient.files, + )({ + file: new File([input], "batch.jsonl"), + purpose: "batch", }); - const created = await batchFixtureClient.batches.create(batchParams); - - const completedAt = Date.now() / 1000 + 60; - const completed = { - ...created, - status: "completed", - in_progress_at: 1_740_000_010, - completed_at: completedAt, - request_counts: { completed: 2, failed: 1, total: 3 }, - }; + const created = await batchFixtureClient.batches.create({ + input_file_id: inputFile.id, + completion_window: "24h", + endpoint: "/v1/chat/completions", + }); + const completed = await openaiBatchesRetrieveTraced( + batchFixtureClient.batches, + )(created.id); // Start both content requests before awaiting either one. These promises // model the APIPromise values returned by files.content(). const outputFile = Promise.resolve( @@ -713,16 +760,11 @@ export async function runOpenAIInstrumentationScenario(options) { ), ); await completeOpenAIBatchTrace({ - batch: completed, - inputFile: input, + inputFileId: completed.input_file_id, + inputFileContent: input, // Batch results are not guaranteed to preserve input order. - outputFile, - errorFile, - webhook: { - type: "batch.completed", - created_at: completedAt, - data: { id: completed.id }, - }, + outputFileContent: outputFile, + errorFileContent: errorFile, }); if (!(await outputFile).bodyUsed || !(await errorFile).bodyUsed) { throw new Error("Expected batch result responses to be consumed"); diff --git a/js/src/exports.ts b/js/src/exports.ts index bd20c477f..91494e404 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -131,7 +131,8 @@ export { registerSandbox } from "./sandbox"; export { completeOpenAIBatchTrace, - startOpenAIBatchTrace, + openaiBatchesRetrieveTraced, + openaiFilesCreateTraced, } from "./openai-batch"; // Internal isomorph layer for platform-specific implementations diff --git a/js/src/instrumentation/plugins/openai-batch-constants.ts b/js/src/instrumentation/plugins/openai-batch-constants.ts deleted file mode 100644 index 5451014b1..000000000 --- a/js/src/instrumentation/plugins/openai-batch-constants.ts +++ /dev/null @@ -1 +0,0 @@ -export const BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY = "braintrust.batch_context"; diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts index e9560bbb1..96a2809ef 100644 --- a/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.test.ts @@ -1,7 +1,6 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { _exportsForTestingOnly, - _internalGetGlobalState, initLogger, startSpan, withCurrent, @@ -9,16 +8,16 @@ import { import { configureNode } from "../../node/config"; import { completeOpenAIBatchTrace, - startOpenAIBatchTrace, + openaiBatchesRetrieveTraced, + openaiFilesCreateTraced, } from "../../openai-batch"; import type { - OpenAIBatchCreateInputParams, - OpenAIBatchJSONL, - OpenAIFileLike, + OpenAIAPIPromise, + OpenAIBatchLike, + OpenAIEnhancedResponse, } from "../../openai-batch-types"; -import { BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY } from "./openai-batch-constants"; -import { OpenAIPlugin } from "./openai-plugin"; import { SpanComponentsV4 } from "../../../util/span_identifier_v4"; +import { OpenAIPlugin } from "./openai-plugin"; try { configureNode(); @@ -47,45 +46,47 @@ const chatInput = [ }, ]; -async function startTestBatchTrace(args: { - input: OpenAIBatchJSONL; - inputFile: OpenAIFileLike; - params: OpenAIBatchCreateInputParams; -}) { - const params = await startOpenAIBatchTrace(args); - return { - id: `batch_${args.inputFile.id}`, - ...params, - status: "validating", +function jsonl(records: unknown[]): string { + return records.map((record) => JSON.stringify(record)).join("\n"); +} + +function apiPromise(data: T): OpenAIAPIPromise { + return asyncAPIPromise(Promise.resolve(data)); +} + +function asyncAPIPromise(dataPromise: Promise): OpenAIAPIPromise { + const promise = dataPromise as OpenAIAPIPromise; + let enhancedResponse: Promise> | undefined; + promise.withResponse = () => { + enhancedResponse ??= dataPromise.then((data) => ({ + data, + response: new Response(JSON.stringify(data), { + headers: { "content-type": "application/json" }, + }), + request_id: "request-id", + })); + return enhancedResponse; }; + promise.asResponse = async () => (await promise.withResponse()).response; + return promise; } -function chunkedJSONLResponse( - records: unknown[], - beforeFirstChunk?: () => Promise, -): Response { - const bytes = new TextEncoder().encode( - records.map((record) => JSON.stringify(record)).join("\n"), - ); +async function uploadedText(upload: unknown): Promise { + if (upload instanceof Blob) { + return await upload.text(); + } + const chunks: Uint8Array[] = []; + for await (const chunk of upload as AsyncIterable) { + chunks.push(chunk); + } + const size = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); + const bytes = new Uint8Array(size); let offset = 0; - return new Response( - new ReadableStream( - { - async pull(controller) { - if (offset === 0) { - await beforeFirstChunk?.(); - } - if (offset < bytes.length) { - controller.enqueue(bytes.slice(offset, offset + 7)); - offset += 7; - } else { - controller.close(); - } - }, - }, - { highWaterMark: 0 }, - ), - ); + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder().decode(bytes); } describe("OpenAI Batch instrumentation", () => { @@ -96,9 +97,6 @@ describe("OpenAI Batch instrumentation", () => { beforeAll(async () => { await _exportsForTestingOnly.simulateLoginForTests(); - _internalGetGlobalState()._internalSetTraceContextSigningSecret( - "openai-batch-test-secret", - ); }); beforeEach(() => { @@ -116,603 +114,284 @@ describe("OpenAI Batch instrumentation", () => { _exportsForTestingOnly.clearTestBackgroundLogger(); }); - it("creates a batch and starts deterministic pending spans", async () => { - const inputFile = { id: "file_input", filename: "batch.jsonl" }; - const input = chatInput.map((record) => ({ - ...record, - body: { ...record.body, user: "not-span-metadata" }, - })); - const sourceParams = { - endpoint: "/v1/chat/completions" as const, - completion_window: "24h" as const, - metadata: { user: "value" }, + it("tees upload streams and starts spans keyed by file ID and custom_id", async () => { + const source = jsonl(chatInput); + let received = ""; + const files = { + marker: "files-resource", + create( + this: { marker: string }, + params: { file: unknown; purpose: string }, + ) { + expect(this.marker).toBe("files-resource"); + return asyncAPIPromise( + Promise.resolve({ id: "file_stream", purpose: params.purpose }).then( + async (file) => { + received = await uploadedText(params.file); + return file; + }, + ), + ); + }, }; - const params = await startTestBatchTrace({ - inputFile, - input: input.map((record) => JSON.stringify(record)).join("\n"), - params: sourceParams, - }); - - expect(params).toMatchObject({ - input_file_id: "file_input", - endpoint: "/v1/chat/completions", - completion_window: "24h", - metadata: { - user: "value", - [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: expect.any(String), + const encoder = new TextEncoder(); + const upload = { + path: "/tmp/batch.jsonl", + async *[Symbol.asyncIterator]() { + yield encoder.encode(source.slice(0, 19)); + yield encoder.encode(source.slice(19)); }, + }; + + const file = await openaiFilesCreateTraced(files)({ + file: upload, + purpose: "batch", }); - expect(sourceParams.metadata).toEqual({ user: "value" }); - expect(inputFile).toEqual({ id: "file_input", filename: "batch.jsonl" }); + expect(file.id).toBe("file_stream"); + expect(received).toBe(source); const rows = (await backgroundLogger.drain()) as Array>; expect(rows).toHaveLength(3); - const task = rows.find( - (row) => row.span_attributes?.name === "openai.batch", - ); + expect( + rows.find((row) => row.span_attributes?.name === "openai.batch"), + ).toMatchObject({ + metadata: { + endpoint: "/v1/chat/completions", + input_file_id: "file_stream", + provider: "openai", + }, + span_attributes: { type: "task" }, + }); const children = rows.filter( (row) => row.span_attributes?.name === "Chat Completion", ); - expect(task).toMatchObject({ - metadata: { provider: "openai" }, - span_attributes: { type: "task" }, - }); - expect(task?.input).toBeUndefined(); - expect(task?.metrics).not.toHaveProperty("end"); - expect(task?.span_parents).toEqual([]); expect(children).toHaveLength(2); - expect(children.map((row) => row.input?.[0]?.content).sort()).toEqual([ + expect(children.map((row) => row.metadata.custom_id).sort()).toEqual([ + "bad", + "ok", + ]); + expect(children.map((row) => row.input[0].content).sort()).toEqual([ "fail", "hi", ]); - expect(children[0]?.metrics).not.toHaveProperty("end"); - expect( - children.find((row) => row.input?.[0]?.content === "hi"), - ).toMatchObject({ - input: [{ role: "user", content: "hi" }], - metadata: { model: "gpt-test", provider: "openai" }, - }); - expect( - children.find((row) => row.input?.[0]?.content === "hi")?.metadata, - ).not.toHaveProperty("user"); + expect(children.every((row) => row.metrics.end === undefined)).toBe(true); }); - it("prepares parameters without calling the OpenAI API", async () => { - const params = await startOpenAIBatchTrace({ - inputFile: { id: "file_prepare" }, - input: [chatInput[0]], - params: { - endpoint: "/v1/chat/completions", - completion_window: "24h", + it("preserves OpenAI promise helpers and passes non-batch files through", async () => { + let calls = 0; + const files = { + create( + params: { file: Blob; purpose: string }, + options?: { timeout: number }, + ) { + calls++; + expect(options).toEqual({ timeout: 10 }); + return apiPromise({ id: `file_${params.purpose}` }); }, - }); + }; + const create = openaiFilesCreateTraced(files); - expect(params).toMatchObject({ - input_file_id: "file_prepare", - endpoint: "/v1/chat/completions", - completion_window: "24h", - metadata: { - [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: expect.any(String), - }, + const traced = create( + { file: new Blob([jsonl([chatInput[0]])]), purpose: "batch" }, + { timeout: 10 }, + ); + expect(typeof traced.withResponse).toBe("function"); + expect(typeof traced.asResponse).toBe("function"); + await expect(traced.withResponse()).resolves.toMatchObject({ + data: { id: "file_batch" }, + request_id: "request-id", }); + + const ordinary = create( + { file: new Blob(["ordinary"]), purpose: "assistants" }, + { timeout: 10 }, + ); + await expect(ordinary).resolves.toEqual({ id: "file_assistants" }); + expect(calls).toBe(2); expect(await backgroundLogger.drain()).toHaveLength(2); }); - it("keeps separate batch operations distinct when they reuse an input file", async () => { - const args = { - inputFile: { id: "file_reused" }, - input: [chatInput[0]], - params: { - endpoint: "/v1/chat/completions" as const, - completion_window: "24h" as const, + it("uses stable span IDs when the same file is observed again", async () => { + let uploadCount = 0; + const files = { + create() { + uploadCount++; + return apiPromise({ id: "file_stable" }); }, }; + const create = openaiFilesCreateTraced(files); + const params = { + file: new Blob([jsonl([chatInput[0]])]), + purpose: "batch", + }; - const first = await startTestBatchTrace(args); - const firstRows = (await backgroundLogger.drain()) as Array< + await create(params); + const first = (await backgroundLogger.drain()) as Array< Record >; - const second = await startTestBatchTrace(args); - const secondRows = (await backgroundLogger.drain()) as Array< + await create(params); + const second = (await backgroundLogger.drain()) as Array< Record >; - expect(first.metadata?.[BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]).not.toBe( - second.metadata?.[BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY], + expect(uploadCount).toBe(2); + expect(second.map((row) => row.id).sort()).toEqual( + first.map((row) => row.id).sort(), ); - expect(firstRows).toHaveLength(2); - expect(secondRows).toHaveLength(2); - const firstIds = new Set(firstRows.map((row) => row.id)); - expect(secondRows.some((row) => firstIds.has(row.id))).toBe(false); }); - it("signs and verifies batch context with WebCrypto", async () => { - const params = await startTestBatchTrace({ - inputFile: { id: "file_webcrypto" }, - input: [chatInput[0]], - params: { - endpoint: "/v1/chat/completions", - completion_window: "24h", - }, - }); - expect(params.metadata).toHaveProperty(BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY); - await backgroundLogger.drain(); - - await completeOpenAIBatchTrace({ - batch: { - id: "batch_webcrypto", - endpoint: params.endpoint, - input_file_id: params.input_file_id, - status: "completed", - completed_at: 200, - metadata: params.metadata, - }, - inputFile: [chatInput[0]], - outputFile: [ - { - custom_id: "ok", - response: { - status_code: 200, - body: { choices: [], usage: { total_tokens: 0 } }, - }, + it("retrieves batches through the resource and corrects lifecycle timestamps", async () => { + const parent = startSpan({ name: "parent" }); + await withCurrent(parent, async () => { + await openaiFilesCreateTraced({ + create() { + return apiPromise({ id: "file_timestamps" }); }, - ], + })({ file: new Blob([jsonl(chatInput)]), purpose: "batch" }); }); - expect(await backgroundLogger.drain()).toHaveLength(2); - }); - - it("does not send propagated span event data to OpenAI", async () => { - const parent = startSpan({ - name: "sensitive-parent", - propagatedEvent: { metadata: { secret: "do-not-send" } }, - }); - const batch = await withCurrent(parent, () => - startTestBatchTrace({ - inputFile: { id: "file_minimal_parent" }, - input: [chatInput[0]], - params: { - endpoint: "/v1/chat/completions", - completion_window: "24h", - }, - }), - ); - parent.end(); - - const serialized = JSON.parse( - batch.metadata?.[BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY] ?? "null", - ) as { parent?: unknown; version?: unknown } | null; - expect(serialized?.version).toBe(1); - expect(typeof serialized?.parent).toBe("string"); - const exported = SpanComponentsV4.fromStr(serialized?.parent as string); - expect(exported.data.propagated_event).toBeUndefined(); - }); - - it("handles file promise rejections on early completion paths", async () => { - const batch = { - id: "batch_in_progress", + await backgroundLogger.drain(); + const batch: OpenAIBatchLike = { + id: "batch_timestamps", endpoint: "/v1/chat/completions", - input_file_id: "file_input", - status: "in_progress", - }; - - await expect( - completeOpenAIBatchTrace({ - batch, - inputFile: chatInput, - outputFile: Promise.reject(new Error("output unavailable")), - errorFile: Promise.reject(new Error("errors unavailable")), - }), - ).resolves.toBe(batch); - await new Promise((resolve) => queueMicrotask(resolve)); - expect(await backgroundLogger.drain()).toEqual([]); - }); - - it("completes successful and failed spans from resumable files", async () => { - const params = await startTestBatchTrace({ - inputFile: { id: "file_input" }, - input: chatInput, - params: { - endpoint: "/v1/chat/completions", - completion_window: "24h", - }, - }); - const pendingRows = (await backgroundLogger.drain()) as Array< - Record - >; - const pendingIds = pendingRows.map((row) => row.id).sort(); - const batch = { - id: "batch_complete", - endpoint: params.endpoint, - input_file_id: params.input_file_id, - completion_window: params.completion_window, + input_file_id: "file_timestamps", status: "completed", - completed_at: 140, - request_counts: { completed: 1, failed: 1, total: 2 }, - metadata: params.metadata, + created_at: 100, + in_progress_at: 110, + completed_at: 200, }; - const outputRecords = [ - { - custom_id: "ok", - response: { - status_code: 200, - body: { - choices: [ - { - index: 0, - message: { role: "assistant", content: "hello 👋" }, - }, - ], - model: "gpt-resolved", - usage: { prompt_tokens: 2, completion_tokens: 3, total_tokens: 5 }, - }, - }, + const batches = { + marker: "batches-resource", + retrieve(this: { marker: string }, batchId: string) { + expect(this.marker).toBe("batches-resource"); + expect(batchId).toBe("batch_timestamps"); + return apiPromise(batch); }, - ]; - const errorRecords = [ - { custom_id: "bad", error: { code: "server_error", message: "failed" } }, - ]; - let readersStarted = 0; - let releaseReaders: (() => void) | undefined; - const bothReadersStarted = new Promise((resolve) => { - releaseReaders = resolve; - }); - const waitForBothReaders = async () => { - readersStarted++; - if (readersStarted === 2) { - releaseReaders?.(); - } - await bothReadersStarted; }; - const outputResponse = chunkedJSONLResponse( - outputRecords, - waitForBothReaders, - ); - const errorResponse = chunkedJSONLResponse( - errorRecords, - waitForBothReaders, - ); - const inputResponse = chunkedJSONLResponse(chatInput); - const completedAt = Date.now() / 1000 + 60; - batch.completed_at = completedAt; - await expect( - completeOpenAIBatchTrace({ - batch, - inputFile: Promise.resolve(inputResponse), - outputFile: Promise.resolve(outputResponse), - errorFile: Promise.resolve(errorResponse), - }), - ).resolves.toBe(batch); - expect(readersStarted).toBe(2); - expect(inputResponse.bodyUsed).toBe(true); - expect(outputResponse.bodyUsed).toBe(true); - expect(errorResponse.bodyUsed).toBe(true); + const result = await openaiBatchesRetrieveTraced(batches)(batch.id); + const parentComponents = SpanComponentsV4.fromStr(await parent.export()); + parent.end(); + + expect(result).toBe(batch); const rows = (await backgroundLogger.drain()) as Array>; - expect(rows).toHaveLength(3); - expect(rows.map((row) => row.id).sort()).toEqual(pendingIds); const task = rows.find( (row) => row.span_attributes?.name === "openai.batch", ); const children = rows.filter( (row) => row.span_attributes?.name === "Chat Completion", ); - expect(task).toMatchObject({ - metrics: { end: completedAt }, - }); - expect(task?.output).toBeUndefined(); - expect(children.find((row) => row.output !== undefined)).toMatchObject({ - metadata: { model: "gpt-resolved", provider: "openai" }, - metrics: { - completion_tokens: 3, - end: completedAt, - prompt_tokens: 2, - tokens: 5, - }, - output: [ - { index: 0, message: { role: "assistant", content: "hello 👋" } }, - ], - }); - expect(children.find((row) => row.error !== undefined)?.error).toContain( - "failed", - ); + expect(task?.metrics).toMatchObject({ start: 100, end: 200 }); + expect(task?.span_parents).toEqual([parentComponents.data.span_id]); + expect(task?.root_span_id).toBe(parentComponents.data.root_span_id); + expect(children).toHaveLength(2); + expect( + children.every( + (row) => row.metrics.start === 110 && row.metrics.end === 200, + ), + ).toBe(true); + }); - await completeOpenAIBatchTrace({ - batch, - inputFile: chatInput, - outputFile: Promise.resolve(chunkedJSONLResponse(outputRecords)), - errorFile: Promise.resolve(chunkedJSONLResponse(errorRecords)), - }); - const replayRows = (await backgroundLogger.drain()) as Array< + it("completes result spans by custom_id and consumes supplied responses", async () => { + await openaiFilesCreateTraced({ + create() { + return apiPromise({ id: "file_results" }); + }, + })({ file: new Blob([jsonl(chatInput)]), purpose: "batch" }); + const pendingRows = (await backgroundLogger.drain()) as Array< Record >; - expect(replayRows.map((row) => row.id).sort()).toEqual(pendingIds); - }); - - it("leaves a completed batch pending until all results are available", async () => { - const batch = await startTestBatchTrace({ - inputFile: { id: "file_incomplete" }, - input: chatInput, - params: { - endpoint: "/v1/chat/completions", - completion_window: "24h", + const pendingIds = pendingRows.map((row) => row.id).sort(); + await openaiBatchesRetrieveTraced({ + retrieve() { + return apiPromise({ + id: "batch_results", + endpoint: "/v1/chat/completions", + input_file_id: "file_results", + status: "completed", + created_at: 100, + in_progress_at: 110, + completed_at: 200, + }); }, - }); + })("batch_results"); await backgroundLogger.drain(); - - await completeOpenAIBatchTrace({ - batch: { - ...batch, - status: "completed", - completed_at: 400, - request_counts: { completed: 1, failed: 1, total: 2 }, - }, - inputFile: chatInput, - outputFile: [ + const inputFileContent = new Response(jsonl(chatInput)); + const outputFileContent = new Response( + jsonl([ { custom_id: "ok", response: { status_code: 200, - body: { choices: [], model: "gpt-resolved" }, + body: { + choices: [ + { + index: 0, + message: { role: "assistant", content: "hello" }, + }, + ], + model: "gpt-result", + usage: { + prompt_tokens: 2, + completion_tokens: 3, + total_tokens: 5, + }, + }, }, }, - ], - }); - - const rows = (await backgroundLogger.drain()) as Array>; - const task = rows.find( - (row) => row.span_attributes?.name === "openai.batch", + ]), ); - expect(task?.metrics).not.toHaveProperty("end"); - expect(task?.output).toBeUndefined(); - }); - - it("rejects completion input that does not match the signed batch input", async () => { - const params = await startTestBatchTrace({ - inputFile: { id: "file_mismatched" }, - input: chatInput, - params: { - endpoint: "/v1/chat/completions", - completion_window: "24h", - }, - }); - await backgroundLogger.drain(); - - await completeOpenAIBatchTrace({ - batch: { - id: "batch_mismatched", - endpoint: params.endpoint, - input_file_id: params.input_file_id, - status: "completed", - completed_at: 400, - request_counts: { completed: 1, failed: 1, total: 2 }, - metadata: params.metadata, - }, - inputFile: chatInput.map((record) => ({ - ...record, - body: { - ...record.body, - messages: [{ role: "user", content: `changed-${record.custom_id}` }], - }, - })), - outputFile: [ + const errorFileContent = new Response( + jsonl([ { - custom_id: "ok", - response: { - status_code: 200, - body: { choices: [], model: "gpt-resolved" }, - }, + custom_id: "bad", + error: { code: "server_error", message: "failed" }, }, - ], - errorFile: [{ custom_id: "bad", error: { message: "expected failure" } }], - }); - - expect(await backgroundLogger.drain()).toEqual([]); - }); - - it("falls back from invalid provider terminal timestamps", async () => { - const params = await startTestBatchTrace({ - inputFile: { id: "file_invalid_timestamp" }, - input: [chatInput[0]], - params: { - endpoint: "/v1/chat/completions", - completion_window: "24h", - }, - }); - await backgroundLogger.drain(); - const beforeCompletion = Date.now() / 1000; + ]), + ); await completeOpenAIBatchTrace({ - batch: { - id: "batch_invalid_timestamp", - endpoint: params.endpoint, - input_file_id: params.input_file_id, - status: "completed", - completed_at: beforeCompletion - 1, - request_counts: { completed: 1, failed: 0, total: 1 }, - metadata: params.metadata, - }, - inputFile: [chatInput[0]], - outputFile: [ - { - custom_id: "ok", - response: { - status_code: 200, - body: { choices: [], model: "gpt-resolved" }, - }, - }, - ], - webhook: { - type: "batch.completed", - created_at: beforeCompletion - 1, - data: { id: "batch_invalid_timestamp" }, - }, + inputFileId: "file_results", + inputFileContent, + outputFileContent, + errorFileContent, }); + expect(inputFileContent.bodyUsed).toBe(true); + expect(outputFileContent.bodyUsed).toBe(true); + expect(errorFileContent.bodyUsed).toBe(true); const rows = (await backgroundLogger.drain()) as Array>; - for (const row of rows) { - expect(row.metrics?.end).toEqual(expect.any(Number)); - expect(row.metrics.end).toBeGreaterThanOrEqual(beforeCompletion); - } - }); - - it("maps Responses records from async input and validates webhook context", async () => { - async function* input() { - yield { - custom_id: "response_1", - method: "POST", - url: "/v1/responses", - body: { model: "gpt-test", input: "hello" }, - }; - } - const params = await startTestBatchTrace({ - inputFile: { id: "file_responses" }, - input: input(), - params: { endpoint: "/v1/responses", completion_window: "24h" }, - }); - await backgroundLogger.drain(); - const batch = { - id: "batch_responses", - endpoint: params.endpoint, - input_file_id: params.input_file_id, - status: "completed", - completed_at: 260, - request_counts: { completed: 1, failed: 0, total: 1 }, - metadata: params.metadata, - }; - const outputFile = [ - { - custom_id: "response_1", - response: { - status_code: 200, - body: { - id: "resp_1", - model: "gpt-test", - output: [{ type: "message", role: "assistant", content: [] }], - usage: { input_tokens: 4, output_tokens: 6, total_tokens: 10 }, - }, - }, - }, - ]; - const completedAt = Date.now() / 1000 + 60; - batch.completed_at = completedAt; - - await completeOpenAIBatchTrace({ - batch, - inputFile: input(), - outputFile, - webhook: { - type: "batch.completed", - created_at: 260, - data: { id: batch.id }, - }, - }); - const rows = (await backgroundLogger.drain()) as Array>; - const responseSpan = rows.find( - (row) => row.span_attributes?.name === "openai.responses.create", - ); - expect(responseSpan).toMatchObject({ - metadata: { - model: "gpt-test", - provider: "openai", - }, + expect(rows.map((row) => row.id).sort()).toEqual(pendingIds); + expect(rows.find((row) => row.output !== undefined)).toMatchObject({ + metadata: { model: "gpt-result", provider: "openai" }, metrics: { - completion_tokens: 6, - end: completedAt, - prompt_tokens: 4, - tokens: 10, - }, - output: [{ type: "message", role: "assistant", content: [] }], - }); - expect(responseSpan?.metrics).not.toHaveProperty("time_to_first_token"); - - await completeOpenAIBatchTrace({ - batch, - inputFile: input(), - outputFile, - webhook: { type: "batch.completed", data: { id: "another_batch" } }, - }); - expect(await backgroundLogger.drain()).toEqual([]); - }); - - it("ends a result span when output processing throws", async () => { - const params = await startTestBatchTrace({ - inputFile: { id: "file_invalid_output" }, - input: [ - { - custom_id: "response_1", - method: "POST", - url: "/v1/responses", - body: { model: "gpt-test", input: "hello" }, - }, - ], - params: { endpoint: "/v1/responses", completion_window: "24h" }, - }); - await backgroundLogger.drain(); - - const output = new Proxy([], { - get(target, property, receiver) { - if (property === "map") { - throw new Error("could not process output"); - } - return Reflect.get(target, property, receiver); - }, - }); - const completedAt = Date.now() / 1000 + 60; - await completeOpenAIBatchTrace({ - batch: { - id: "batch_invalid_output", - endpoint: params.endpoint, - input_file_id: params.input_file_id, - status: "completed", - completed_at: completedAt, - metadata: params.metadata, + completion_tokens: 3, + end: 200, + prompt_tokens: 2, + tokens: 5, }, - inputFile: [ - { - custom_id: "response_1", - method: "POST", - url: "/v1/responses", - body: { model: "gpt-test", input: "hello" }, - }, - ], - outputFile: [ - { - custom_id: "response_1", - response: { - status_code: 200, - body: { output }, - }, - }, - ], + output: [{ index: 0, message: { role: "assistant", content: "hello" } }], }); - - const rows = (await backgroundLogger.drain()) as Array>; - const child = rows.find( - (row) => row.span_attributes?.name === "openai.responses.create", + expect(rows.find((row) => row.error !== undefined)?.error).toContain( + "failed", ); - expect(child?.error).toContain("could not process output"); - expect(child?.metrics?.end).toBe(completedAt); }); - it("leaves conflicting reserved metadata untouched", async () => { - let inputReads = 0; - async function* input() { - inputReads++; - yield* chatInput; - } - const params = await startTestBatchTrace({ - inputFile: { id: "file_input" }, - input: input(), - params: { - endpoint: "/v1/responses", - completion_window: "24h", - metadata: { [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: "user-owned" }, + it("does not interfere with uploads when the JSONL is invalid", async () => { + const content = "not jsonl"; + let received = ""; + const result = await openaiFilesCreateTraced({ + create(params: { file: Blob; purpose: string }) { + return asyncAPIPromise( + Promise.resolve({ id: "file_invalid" }).then(async (file) => { + received = await params.file.text(); + return file; + }), + ); }, - }); - expect(params.metadata).toEqual({ - [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: "user-owned", - }); - expect(inputReads).toBe(0); + })({ file: new Blob([content]), purpose: "batch" }); + + expect(result).toEqual({ id: "file_invalid" }); + expect(received).toBe(content); expect(await backgroundLogger.drain()).toEqual([]); }); }); diff --git a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts index b4bc0cafc..560ac51ca 100644 --- a/js/src/instrumentation/plugins/openai-batch-instrumentation.ts +++ b/js/src/instrumentation/plugins/openai-batch-instrumentation.ts @@ -1,6 +1,5 @@ import { debugLogger } from "../../debug-logger"; import { - _internalGetGlobalState, _internalStartSpanWithInitialMerge, _internalStartSpanWithInitialMergeAndParentSpanIds, getSpanParentObject, @@ -14,19 +13,13 @@ import { withSpanInstrumentationName, } from "../../span-origin"; import { getCurrentUnixTimestamp } from "../../util"; -import { SpanTypeAttribute, isObject } from "../../../util/index"; -import { - SpanComponentsV4, - type SpanComponentsV4Data, -} from "../../../util/span_identifier_v4"; +import { isObject, SpanTypeAttribute } from "../../../util/index"; +import { SpanComponentsV4 } from "../../../util/span_identifier_v4"; import type { CompleteOpenAIBatchTraceArgs, - OpenAIBatchCreateParams, - OpenAIBatchJSONL, + OpenAIBatchFile, OpenAIBatchLike, - StartOpenAIBatchTraceArgs, } from "../../openai-batch-types"; -import { BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY } from "./openai-batch-constants"; import { openAIChannels } from "./openai-channels"; import { extractOpenAIBatchInput, @@ -40,20 +33,31 @@ const TERMINAL_STATUSES = new Set([ "expired", "cancelled", ]); -const INPUT_DIGEST_CHUNK_SIZE = 1024 * 1024; -type BatchContext = { - version: 1; - parent: string; +type BatchInputRecord = { + customId: string; + spanData?: ReturnType; +}; + +type BatchInputs = { + endpoint?: string; + inputs: Map; + issues: Error[]; +}; + +type BatchTraceContext = { inputFileId: string; - inputDigest: string; endpoint: string; - startTime: number; - operationNonce: string; + parent: string; + taskStartTime: number; + childStartTime: number; }; -type SerializedBatchContext = Omit & { - signature: string; +type PendingBatchTrace = { + context: BatchTraceContext; + inputs: Map; + endTime?: number; + status?: string; }; type BatchResultRecord = { @@ -61,19 +65,7 @@ type BatchResultRecord = { source: "output" | "error"; }; -type BatchInputRecord = { - customId: string; - spanData?: ReturnType; -}; - -type OpenAIBatchFile = - CompleteOpenAIBatchTraceArgs["outputFile"]; - -type PreparedBatch = { - context: BatchContext; - inputs: Map; - params: OpenAIBatchCreateParams; -}; +const pendingBatchTraces = new Map(); function read(value: unknown, key: PropertyKey): unknown { if (!isObject(value)) { @@ -95,14 +87,10 @@ function isBatchRecordIterable( ) { return false; } - try { - return ( - typeof Reflect.get(value, Symbol.iterator) === "function" || - typeof Reflect.get(value, Symbol.asyncIterator) === "function" - ); - } catch { - return false; - } + return ( + typeof read(value, Symbol.iterator) === "function" || + typeof read(value, Symbol.asyncIterator) === "function" + ); } function validCustomId(value: unknown): value is string { @@ -113,293 +101,16 @@ function logBatchInstrumentationError(context: string, error: unknown): void { debugLogger.debug(`OpenAI Batch instrumentation ${context}:`, error); } -function batchContextPayload(context: BatchContext): string { - return JSON.stringify([BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY, context]); -} - -async function importBatchSigningKey(secret: string) { - return await globalThis.crypto.subtle.importKey( - "raw", - new TextEncoder().encode(secret), - { name: "HMAC", hash: "SHA-256" }, - false, - ["sign", "verify"], - ); -} - -async function signBatchContext( - context: BatchContext, - secret: string, +async function exportParent( + parent: { export(): Promise } | { toStr(): string }, ): Promise { - try { - const signature = await globalThis.crypto.subtle.sign( - "HMAC", - await importBatchSigningKey(secret), - new TextEncoder().encode(batchContextPayload(context)), - ); - return digestHex(new Uint8Array(signature), 32); - } catch { - return undefined; - } -} - -function signatureBytes(value: string): Uint8Array | undefined { - if (!/^[0-9a-f]{64}$/.test(value)) { - return undefined; - } - return Uint8Array.from(value.match(/.{2}/g) ?? [], (byte) => - Number.parseInt(byte, 16), - ); -} - -async function verifyBatchContext( - context: BatchContext, - signature: string, -): Promise { - const secret = - _internalGetGlobalState()._internalGetTraceContextSigningSecret(); - if (!secret) { - return false; - } - const bytes = signatureBytes(signature); - if (!bytes) { - return false; - } - try { - return await globalThis.crypto.subtle.verify( - "HMAC", - await importBatchSigningKey(secret), - bytes, - new TextEncoder().encode(batchContextPayload(context)), - ); - } catch { - return false; - } -} - -async function parseBatchContext( - value: unknown, - inputFileId: string, - endpoint: string, -): Promise { - if (typeof value !== "string") { - return undefined; - } - - let parsed: unknown; - try { - parsed = JSON.parse(value); - } catch { - return undefined; - } - if ( - !isObject(parsed) || - parsed.version !== 1 || - typeof parsed.parent !== "string" || - typeof parsed.inputDigest !== "string" || - !/^[0-9a-f]{64}$/.test(parsed.inputDigest) || - typeof parsed.startTime !== "number" || - !Number.isFinite(parsed.startTime) || - typeof parsed.operationNonce !== "string" || - !/^[0-9a-f]{32}$/.test(parsed.operationNonce) || - typeof parsed.signature !== "string" - ) { - return undefined; - } - - const context: BatchContext = { - version: 1, - parent: parsed.parent, - inputFileId, - inputDigest: parsed.inputDigest, - endpoint, - startTime: parsed.startTime, - operationNonce: parsed.operationNonce, - }; - if (!(await verifyBatchContext(context, parsed.signature))) { - return undefined; - } - - try { - SpanComponentsV4.fromStr(context.parent); - } catch { - return undefined; - } - return context; -} - -async function exportParent(parent: ReturnType) { if ("toStr" in parent && typeof parent.toStr === "function") { return parent.toStr(); } - return await parent.export(); -} - -async function exportMinimalParent( - parent: ReturnType, -): Promise { - const exported = await exportParent(parent); - if (!exported) { - return undefined; - } - const parsed = SpanComponentsV4.fromStr(exported).data; - const projectId = read(parsed.compute_object_metadata_args, "project_id"); - const projectName = read(parsed.compute_object_metadata_args, "project_name"); - const computeObjectMetadataArgs = - typeof projectId === "string" || typeof projectName === "string" - ? { - ...(typeof projectId === "string" ? { project_id: projectId } : {}), - ...(typeof projectName === "string" - ? { project_name: projectName } - : {}), - } - : undefined; - if (!parsed.object_id && !computeObjectMetadataArgs) { - return undefined; - } - - const routing = parsed.object_id - ? { object_id: parsed.object_id } - : { compute_object_metadata_args: computeObjectMetadataArgs ?? {} }; - // SpanComponents requires all three span identifiers together. The - // conditional below preserves that invariant, but TypeScript cannot infer - // it through the object spreads. - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return new SpanComponentsV4({ - object_type: parsed.object_type, - ...routing, - ...(parsed.row_id && parsed.span_id && parsed.root_span_id - ? { - row_id: parsed.row_id, - span_id: parsed.span_id, - root_span_id: parsed.root_span_id, - } - : {}), - } as SpanComponentsV4Data).toStr(); -} - -async function prepareCreateParams( - args: StartOpenAIBatchTraceArgs, - parent: ReturnType, - startTime: number, -): Promise { - const params: OpenAIBatchCreateParams = { - ...args.params, - input_file_id: args.inputFile.id, - }; - const endpoint = read(params, "endpoint"); - const inputFileId = read(params, "input_file_id"); - if ( - typeof endpoint !== "string" || - !SUPPORTED_ENDPOINTS.has(endpoint) || - typeof inputFileId !== "string" || - inputFileId.length === 0 - ) { - return undefined; + if ("export" in parent && typeof parent.export === "function") { + return await parent.export(); } - - const metadataValue = read(params, "metadata"); - if ( - metadataValue !== undefined && - metadataValue !== null && - (!isObject(metadataValue) || Array.isArray(metadataValue)) - ) { - logBatchInstrumentationError( - "skipped context injection", - "invalid metadata", - ); - return undefined; - } - const metadata = metadataValue ?? {}; - if ( - Object.prototype.hasOwnProperty.call( - metadata, - BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY, - ) - ) { - logBatchInstrumentationError( - "skipped context injection", - "conflicting reserved metadata", - ); - return undefined; - } - if (Object.keys(metadata).length >= 16) { - logBatchInstrumentationError( - "skipped context injection", - "metadata is full", - ); - return undefined; - } - - const exportedParent = await exportMinimalParent(parent); - if (!exportedParent) { - logBatchInstrumentationError( - "skipped context injection", - "no routable Braintrust parent", - ); - return undefined; - } - const signingSecret = - _internalGetGlobalState()._internalGetTraceContextSigningSecret(); - if (!signingSecret) { - logBatchInstrumentationError( - "skipped context injection", - "no signing secret is available", - ); - return undefined; - } - const inputData = await readBatchInputs(args.input, endpoint); - if (inputData.issues.length > 0) { - logBatchInstrumentationError( - "skipped invalid input file", - inputData.issues[0], - ); - return undefined; - } - const context: BatchContext = { - version: 1, - parent: exportedParent, - inputFileId, - inputDigest: inputData.inputDigest, - endpoint, - startTime, - operationNonce: randomBatchNonce(), - }; - const signature = await signBatchContext(context, signingSecret); - if (!signature) { - logBatchInstrumentationError( - "skipped context injection", - "could not sign batch context", - ); - return undefined; - } - const serialized = JSON.stringify({ - version: context.version, - parent: context.parent, - inputDigest: context.inputDigest, - startTime: context.startTime, - operationNonce: context.operationNonce, - signature, - } satisfies SerializedBatchContext); - if (serialized.length > 512) { - logBatchInstrumentationError( - "skipped context injection", - "signed context exceeds OpenAI metadata limits", - ); - return undefined; - } - - return { - context, - inputs: inputData.inputs, - params: { - ...params, - metadata: { - ...metadata, - [BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY]: serialized, - }, - }, - }; + return undefined; } async function deterministicDigest(namespace: string, ...parts: string[]) { @@ -422,19 +133,15 @@ function digestUuid(bytes: Uint8Array): string { return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; } -function randomBatchNonce(): string { - return digestHex(globalThis.crypto.getRandomValues(new Uint8Array(16)), 16); -} - -async function batchSpanIds(operationNonce: string): Promise<{ +async function batchSpanIds(inputFileId: string): Promise<{ rowId: string; spanId: string; rootSpanId: string; }> { const [row, span, root] = await Promise.all([ - deterministicDigest("openai:batch:row", operationNonce), - deterministicDigest("openai:batch:span", operationNonce), - deterministicDigest("openai:batch:root", operationNonce), + deterministicDigest("openai:batch:row", inputFileId), + deterministicDigest("openai:batch:span", inputFileId), + deterministicDigest("openai:batch:root", inputFileId), ]); return { rowId: digestUuid(row), @@ -443,36 +150,16 @@ async function batchSpanIds(operationNonce: string): Promise<{ }; } -async function childSpanIds(operationNonce: string, customId: string) { +async function childSpanIds(inputFileId: string, customId: string) { const [row, span] = await Promise.all([ - deterministicDigest("openai:batch:child:row", operationNonce, customId), - deterministicDigest("openai:batch:child:span", operationNonce, customId), + deterministicDigest("openai:batch:child:row", inputFileId, customId), + deterministicDigest("openai:batch:child:span", inputFileId, customId), ]); return { rowId: digestUuid(row), spanId: digestHex(span, 8) }; } -async function batchContextFromBatch( - batch: OpenAIBatchLike, -): Promise { - const metadata = read(batch, "metadata"); - const inputFileId = read(batch, "input_file_id"); - const endpoint = read(batch, "endpoint"); - if ( - !isObject(metadata) || - typeof inputFileId !== "string" || - typeof endpoint !== "string" - ) { - return undefined; - } - return await parseBatchContext( - read(metadata, BRAINTRUST_OPENAI_BATCH_CONTEXT_KEY), - inputFileId, - endpoint, - ); -} - -async function startBatchSpan(context: BatchContext): Promise { - const ids = await batchSpanIds(context.operationNonce); +async function startBatchSpan(context: BatchTraceContext): Promise { + const ids = await batchSpanIds(context.inputFileId); const parent = SpanComponentsV4.fromStr(context.parent); const hasParentSpan = Boolean( parent.data.row_id && parent.data.span_id && parent.data.root_span_id, @@ -493,10 +180,14 @@ async function startBatchSpan(context: BatchContext): Promise { } : {}), spanId: ids.spanId, - startTime: context.startTime, + startTime: context.taskStartTime, event: { id: ids.rowId, - metadata: { provider: "openai" }, + metadata: { + endpoint: context.endpoint, + input_file_id: context.inputFileId, + provider: "openai", + }, }, }, INSTRUMENTATION_NAMES.OPENAI, @@ -506,11 +197,11 @@ async function startBatchSpan(context: BatchContext): Promise { } async function startBatchChild( - context: BatchContext, + context: BatchTraceContext, taskParent: string, input: BatchInputRecord, ): Promise { - const ids = await childSpanIds(context.operationNonce, input.customId); + const ids = await childSpanIds(context.inputFileId, input.customId); return withCurrent(NOOP_SPAN, () => _internalStartSpanWithInitialMerge( withSpanInstrumentationName( @@ -522,7 +213,7 @@ async function startBatchChild( type: SpanTypeAttribute.LLM, parent: taskParent, spanId: ids.spanId, - startTime: context.startTime, + startTime: context.childStartTime, event: { id: ids.rowId, ...(input.spanData?.input !== undefined @@ -530,6 +221,7 @@ async function startBatchChild( : {}), metadata: { ...input.spanData?.metadata, + custom_id: input.customId, provider: "openai", }, }, @@ -541,25 +233,17 @@ async function startBatchChild( } async function* jsonlRecords( - file: OpenAIBatchJSONL | OpenAIBatchFile, + file: OpenAIBatchFile, onIssue: (error: Error) => void = () => {}, ): AsyncGenerator { const resolvedFile = await file; - if (resolvedFile === undefined) { - return; - } if (typeof resolvedFile === "string") { - let offset = 0; - while (offset < resolvedFile.length) { - const newline = resolvedFile.indexOf("\n", offset); - const end = newline === -1 ? resolvedFile.length : newline; - const line = resolvedFile.slice(offset, end).replace(/\r$/, ""); - offset = newline === -1 ? resolvedFile.length : newline + 1; + for (const line of resolvedFile.split("\n")) { if (!line.trim()) { continue; } try { - yield JSON.parse(line); + yield JSON.parse(line.replace(/\r$/, "")); } catch (error) { logBatchInstrumentationError("skipped malformed JSONL", error); onIssue(new Error("OpenAI Batch file contains malformed JSONL")); @@ -574,16 +258,12 @@ async function* jsonlRecords( let reader: unknown; try { reader = Reflect.apply(getReader, body, []); - if (!isObject(reader)) { - throw new Error("Response body returned an invalid stream reader"); - } - const decoder = new TextDecoder(); let pending = ""; while (true) { const readChunk = read(reader, "read"); if (typeof readChunk !== "function") { - throw new Error("Response body stream reader has no read method"); + throw new Error("Response body stream has no read method"); } const chunk = await Reflect.apply(readChunk, reader, []); if (!isObject(chunk)) { @@ -596,7 +276,6 @@ async function* jsonlRecords( if (!(chunk.value instanceof Uint8Array)) { throw new Error("Response body stream returned a non-byte chunk"); } - pending += decoder.decode(chunk.value, { stream: true }); let newline = pending.indexOf("\n"); while (newline !== -1) { @@ -607,21 +286,18 @@ async function* jsonlRecords( yield JSON.parse(line); } catch (error) { logBatchInstrumentationError("skipped malformed JSONL", error); - onIssue( - new Error("OpenAI Batch response contains malformed JSONL"), - ); + onIssue(new Error("OpenAI Batch file contains malformed JSONL")); } } newline = pending.indexOf("\n"); } } - if (pending.trim()) { try { yield JSON.parse(pending.replace(/\r$/, "")); } catch (error) { logBatchInstrumentationError("skipped malformed JSONL", error); - onIssue(new Error("OpenAI Batch response contains malformed JSONL")); + onIssue(new Error("OpenAI Batch file contains malformed JSONL")); } } } catch (error) { @@ -629,7 +305,7 @@ async function* jsonlRecords( onIssue(new Error("OpenAI Batch response body could not be read")); } finally { const releaseLock = read(reader, "releaseLock"); - if (isObject(reader) && typeof releaseLock === "function") { + if (typeof releaseLock === "function") { try { Reflect.apply(releaseLock, reader, []); } catch (error) { @@ -647,56 +323,177 @@ async function* jsonlRecords( for await (const record of resolvedFile) { yield record; } - } else { - logBatchInstrumentationError("skipped invalid JSONL source", resolvedFile); - onIssue(new Error("OpenAI Batch file source is invalid")); + return; } + + logBatchInstrumentationError("skipped invalid JSONL source", resolvedFile); + onIssue(new Error("OpenAI Batch file source is invalid")); } -async function startPreparedBatch( - inputs: Map, - context: BatchContext, +async function readBatchInputs(file: OpenAIBatchFile): Promise { + const inputs = new Map(); + const issues: Error[] = []; + let endpoint: string | undefined; + try { + for await (const value of jsonlRecords(file, (issue) => + issues.push(issue), + )) { + const customId = read(value, "custom_id"); + const url = read(value, "url"); + const body = read(value, "body"); + if ( + !validCustomId(customId) || + inputs.has(customId) || + read(value, "method") !== "POST" || + typeof url !== "string" || + !SUPPORTED_ENDPOINTS.has(url) || + (endpoint !== undefined && endpoint !== url) || + !isObject(body) + ) { + issues.push(new Error("OpenAI Batch input contains an invalid record")); + continue; + } + endpoint = url; + let spanData: BatchInputRecord["spanData"]; + try { + spanData = extractOpenAIBatchInput(url, body); + } catch (error) { + logBatchInstrumentationError("could not extract batch input", error); + } + inputs.set(customId, { customId, spanData }); + } + } catch (error) { + logBatchInstrumentationError("could not process input file", error); + issues.push(new Error("OpenAI Batch input file could not be processed")); + } + if (!endpoint || inputs.size === 0) { + issues.push(new Error("OpenAI Batch input contains no supported records")); + } + return { endpoint, inputs, issues }; +} + +async function writePendingSpans( + trace: PendingBatchTrace, + endTime?: number, ): Promise { - const task = await startBatchSpan(context); + const task = await startBatchSpan(trace.context); const taskParent = await task.export(); - for (const input of inputs.values()) { + for (const input of trace.inputs.values()) { try { - await startBatchChild(context, taskParent, input); + const child = await startBatchChild(trace.context, taskParent, input); + if (endTime !== undefined) { + child.end({ endTime }); + } } catch (error) { - logBatchInstrumentationError("skipped invalid input record", error); + logBatchInstrumentationError("could not write batch request span", error); } } + if (endTime !== undefined) { + if (trace.status && trace.status !== "completed") { + task.log({ error: new Error(`OpenAI Batch ${trace.status}`) }); + } + task.end({ endTime }); + } } -export const interceptOpenAIBatchTraceStart: Parameters< - typeof openAIChannels.batchesStartTrace.intercept +export const interceptOpenAIFilesCreateTraced: Parameters< + typeof openAIChannels.filesCreateTraced.intercept >[0] = async (target, thisArg, args) => { - const parent = getSpanParentObject(); const startTime = getCurrentUnixTimestamp(); - let prepared: PreparedBatch | undefined; + const inputPromise = readBatchInputs(args[0].inputFileContent); + const parentPromise = exportParent(args[0].parent); + const file = await Reflect.apply(target, thisArg, args); try { - prepared = await prepareCreateParams(args[0], parent, startTime); + const inputFileId = read(file, "id"); + const [inputData, exportedParent] = await Promise.all([ + inputPromise, + parentPromise, + ]); + if ( + typeof inputFileId !== "string" || + !exportedParent || + !inputData.endpoint || + inputData.issues.length > 0 + ) { + if (inputData.issues[0]) { + logBatchInstrumentationError( + "skipped invalid input file", + inputData.issues[0], + ); + } + return file; + } + const trace: PendingBatchTrace = { + context: { + inputFileId, + endpoint: inputData.endpoint, + parent: exportedParent, + taskStartTime: startTime, + childStartTime: startTime, + }, + inputs: inputData.inputs, + }; + pendingBatchTraces.set(inputFileId, trace); + await writePendingSpans(trace); } catch (error) { - logBatchInstrumentationError("could not prepare batch", error); + logBatchInstrumentationError("could not start batch spans", error); } + return file; +}; - const params = await Reflect.apply(target, thisArg, [ - prepared - ? { - ...args[0], - params: prepared.params, - } - : args[0], - ]); - if (prepared) { +function validTimestamp(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function terminalEndTime(batch: OpenAIBatchLike, startTime: number): number { + let timestamp: unknown; + switch (batch.status) { + case "completed": + timestamp = batch.completed_at; + break; + case "failed": + timestamp = batch.failed_at; + break; + case "expired": + timestamp = batch.expired_at; + break; + default: + timestamp = batch.cancelled_at; + } + return validTimestamp(timestamp) + ? Math.max(timestamp, startTime) + : Math.max(getCurrentUnixTimestamp(), startTime); +} + +async function updateBatchTimestamps(batch: OpenAIBatchLike): Promise { + const trace = pendingBatchTraces.get(batch.input_file_id); + if (!trace || trace.context.endpoint !== batch.endpoint) { + return; + } + if (validTimestamp(batch.created_at)) { + trace.context.taskStartTime = batch.created_at; + } + trace.context.childStartTime = validTimestamp(batch.in_progress_at) + ? Math.max(batch.in_progress_at, trace.context.taskStartTime) + : trace.context.taskStartTime; + trace.status = batch.status; + if (TERMINAL_STATUSES.has(batch.status)) { + trace.endTime = terminalEndTime(batch, trace.context.childStartTime); + } + await writePendingSpans(trace, trace.endTime); +} + +export const interceptOpenAIBatchesRetrieveTraced: Parameters< + typeof openAIChannels.batchesRetrieveTraced.intercept +>[0] = (target, thisArg, args) => + Promise.resolve(Reflect.apply(target, thisArg, args)).then(async (batch) => { try { - await startPreparedBatch(prepared.inputs, prepared.context); + await updateBatchTimestamps(batch); } catch (error) { - logBatchInstrumentationError("could not start batch spans", error); + logBatchInstrumentationError("could not update batch timestamps", error); } - } - return params; -}; + return batch; + }); function errorFromResult(result: BatchResultRecord): Error | undefined { const error = read(result.value, "error"); @@ -712,9 +509,7 @@ function errorFromResult(result: BatchResultRecord): Error | undefined { result.source === "error" || (typeof statusCode === "number" && (statusCode < 200 || statusCode >= 300)) ) { - const body = read(response, "body"); - const bodyError = read(body, "error"); - const message = read(bodyError, "message"); + const message = read(read(read(response, "body"), "error"), "message"); return new Error( typeof message === "string" ? message : "OpenAI Batch request failed", ); @@ -723,7 +518,7 @@ function errorFromResult(result: BatchResultRecord): Error | undefined { } async function completeBatchResult( - context: BatchContext, + context: BatchTraceContext, taskParent: string, endTime: number, input: BatchInputRecord, @@ -732,26 +527,19 @@ async function completeBatchResult( const child = await startBatchChild(context, taskParent, input); try { const resultError = errorFromResult(result); - const response = read(result.value, "response"); - const responseBody = read(response, "body"); + const responseBody = read(read(result.value, "response"), "body"); if (resultError) { child.log({ error: resultError }); } else if (isObject(responseBody)) { - if (context.endpoint === "/v1/chat/completions") { - const model = read(responseBody, "model"); - child.log({ - output: read(responseBody, "choices"), - ...(typeof model === "string" ? { metadata: { model } } : {}), - metrics: parseMetricsFromUsage(read(responseBody, "usage")), - }); - } else { - const model = read(responseBody, "model"); - child.log({ - output: processImagesInOutput(read(responseBody, "output")), - ...(typeof model === "string" ? { metadata: { model } } : {}), - metrics: parseMetricsFromUsage(read(responseBody, "usage")), - }); - } + const model = read(responseBody, "model"); + child.log({ + output: + context.endpoint === "/v1/chat/completions" + ? read(responseBody, "choices") + : processImagesInOutput(read(responseBody, "output")), + ...(typeof model === "string" ? { metadata: { model } } : {}), + metrics: parseMetricsFromUsage(read(responseBody, "usage")), + }); } else { child.log({ error: new Error("OpenAI Batch response body is missing") }); } @@ -762,133 +550,6 @@ async function completeBatchResult( } } -async function readBatchInputs( - file: OpenAIBatchFile, - endpoint: string, -): Promise<{ - inputDigest: string; - inputs: Map; - issues: Error[]; -}> { - const inputs = new Map(); - const issues: Error[] = []; - const chunkDigests: string[] = []; - let digestChunk = ""; - try { - for await (const value of jsonlRecords(file, (issue) => - issues.push(issue), - )) { - const customId = read(value, "custom_id"); - const body = read(value, "body"); - if ( - !validCustomId(customId) || - inputs.has(customId) || - read(value, "method") !== "POST" || - read(value, "url") !== endpoint || - !isObject(body) - ) { - issues.push(new Error("OpenAI Batch input contains an invalid record")); - continue; - } - const canonicalRecord = JSON.stringify({ - body: canonicalizeJSON(body), - custom_id: customId, - }); - const framedRecord = `${canonicalRecord.length}:${canonicalRecord}`; - if ( - digestChunk.length > 0 && - digestChunk.length + framedRecord.length > INPUT_DIGEST_CHUNK_SIZE - ) { - chunkDigests.push( - digestHex( - await deterministicDigest("openai:batch:input:chunk", digestChunk), - 32, - ), - ); - digestChunk = ""; - } - digestChunk += framedRecord; - - let spanData: BatchInputRecord["spanData"]; - try { - spanData = extractOpenAIBatchInput(endpoint, body); - } catch (error) { - logBatchInstrumentationError("could not extract batch input", error); - } - inputs.set(customId, { customId, spanData }); - } - } catch (error) { - logBatchInstrumentationError("could not process input file", error); - issues.push(new Error("OpenAI Batch input file could not be processed")); - } - chunkDigests.push( - digestHex( - await deterministicDigest("openai:batch:input:chunk", digestChunk), - 32, - ), - ); - return { - inputDigest: digestHex( - await deterministicDigest("openai:batch:input", chunkDigests.join("")), - 32, - ), - inputs, - issues, - }; -} - -function canonicalizeJSON( - value: unknown, - ancestors: Set = new Set(), -): unknown { - if ( - value === null || - typeof value === "string" || - typeof value === "boolean" - ) { - return value; - } - if (typeof value === "number") { - if (!Number.isFinite(value)) { - throw new Error("OpenAI Batch input contains a non-finite number"); - } - return value; - } - if (Array.isArray(value)) { - if (ancestors.has(value)) { - throw new Error("OpenAI Batch input contains a circular value"); - } - ancestors.add(value); - try { - return value.map((entry) => canonicalizeJSON(entry, ancestors)); - } finally { - ancestors.delete(value); - } - } - if (!isObject(value)) { - throw new Error("OpenAI Batch input contains a non-JSON value"); - } - if (ancestors.has(value)) { - throw new Error("OpenAI Batch input contains a circular value"); - } - ancestors.add(value); - try { - return Object.fromEntries( - Object.keys(value) - .sort() - .map((key) => { - const entry = read(value, key); - if (entry === undefined) { - throw new Error("OpenAI Batch input contains an undefined value"); - } - return [key, canonicalizeJSON(entry, ancestors)]; - }), - ); - } finally { - ancestors.delete(value); - } -} - async function completeResultFile({ context, endTime, @@ -899,172 +560,122 @@ async function completeResultFile({ source, taskParent, }: { - context: BatchContext; + context: BatchTraceContext; endTime: number; - file: OpenAIBatchFile; + file: OpenAIBatchFile | undefined; inputs: Map; issues: Error[]; seen: Set; source: BatchResultRecord["source"]; taskParent: string; }): Promise { - try { - for await (const value of jsonlRecords(file, (issue) => - issues.push(issue), - )) { - try { - const customId = read(value, "custom_id"); - if (!validCustomId(customId) || !isObject(value)) { - logBatchInstrumentationError( - "skipped result without custom_id", - value, - ); - issues.push( - new Error("OpenAI Batch result is missing a valid custom_id"), - ); - continue; - } - if (seen.has(customId)) { - logBatchInstrumentationError("skipped duplicate result", customId); - issues.push(new Error("OpenAI Batch result contains a duplicate")); - continue; - } - const input = inputs.get(customId); - if (!input) { - issues.push( - new Error("OpenAI Batch result does not match an input record"), - ); - continue; - } - seen.add(customId); - await completeBatchResult(context, taskParent, endTime, input, { - value, - source, - }); - } catch (error) { - logBatchInstrumentationError("skipped invalid result", error); - issues.push(new Error("OpenAI Batch result could not be processed")); - } + if (file === undefined) { + return; + } + for await (const value of jsonlRecords(file, (issue) => issues.push(issue))) { + const customId = read(value, "custom_id"); + if (!validCustomId(customId) || !isObject(value)) { + issues.push( + new Error("OpenAI Batch result is missing a valid custom_id"), + ); + continue; } - } catch (error) { - logBatchInstrumentationError(`could not process ${source} file`, error); - issues.push( - new Error(`OpenAI Batch ${source} file could not be processed`), - ); + if (seen.has(customId)) { + issues.push(new Error("OpenAI Batch result contains a duplicate")); + continue; + } + const input = inputs.get(customId); + if (!input) { + issues.push( + new Error("OpenAI Batch result does not match an input record"), + ); + continue; + } + seen.add(customId); + await completeBatchResult(context, taskParent, endTime, input, { + value, + source, + }); } } -function terminalEndTime( - batch: OpenAIBatchLike, - webhookCreatedAt: number | undefined, - startTime: number, -): number { - let timestamp: number | null | undefined; - switch (batch.status) { - case "completed": - timestamp = batch.completed_at; - break; - case "failed": - timestamp = batch.failed_at; - break; - case "expired": - timestamp = batch.expired_at; - break; - default: - timestamp = batch.cancelled_at; - } +function sameInputs( + first: Map, + second: Map, +): boolean { + return ( + first.size === second.size && + [...first.keys()].every((customId) => second.has(customId)) + ); +} - if ( - typeof timestamp === "number" && - Number.isFinite(timestamp) && - timestamp >= startTime - ) { - return timestamp; +async function contextForCompletion( + inputFileId: string, + inputData: BatchInputs, +): Promise { + if (!inputData.endpoint || inputData.issues.length > 0) { + return undefined; } - if ( - typeof webhookCreatedAt === "number" && - Number.isFinite(webhookCreatedAt) && - webhookCreatedAt >= startTime - ) { - return webhookCreatedAt; + const existing = pendingBatchTraces.get(inputFileId); + if (existing) { + if ( + existing.context.endpoint !== inputData.endpoint || + !sameInputs(existing.inputs, inputData.inputs) + ) { + return undefined; + } + return { ...existing, inputs: inputData.inputs }; + } + const parent = await exportParent(getSpanParentObject()); + if (!parent) { + return undefined; } - return Math.max(getCurrentUnixTimestamp(), startTime); + const startTime = getCurrentUnixTimestamp(); + return { + context: { + inputFileId, + endpoint: inputData.endpoint, + parent, + taskStartTime: startTime, + childStartTime: startTime, + }, + inputs: inputData.inputs, + }; } async function completeBatch( - args: CompleteOpenAIBatchTraceArgs, + args: CompleteOpenAIBatchTraceArgs, ): Promise { - const { batch, webhook } = args; - const batchId = read(batch, "id"); - const status = read(batch, "status"); - if ( - typeof batchId !== "string" || - typeof status !== "string" || - !TERMINAL_STATUSES.has(status) - ) { - return; - } - if ( - webhook && - (read(webhook, "type") !== `batch.${status}` || - read(read(webhook, "data"), "id") !== batchId) - ) { - logBatchInstrumentationError("ignored mismatched webhook", webhook); - return; - } - const context = await batchContextFromBatch(batch); - if (!context || !SUPPORTED_ENDPOINTS.has(context.endpoint)) { - return; - } - - const endTime = terminalEndTime( - batch, - webhook?.created_at, - context.startTime, - ); - const inputData = await readBatchInputs(args.inputFile, context.endpoint); - const issues = [...inputData.issues]; - if ( - inputData.issues.length === 0 && - inputData.inputDigest !== context.inputDigest - ) { - issues.push( - new Error("OpenAI Batch input does not match the signed input file"), - ); - } - const requestTotal = read(read(batch, "request_counts"), "total"); - if ( - typeof requestTotal === "number" && - requestTotal !== inputData.inputs.size - ) { - issues.push( - new Error("OpenAI Batch input does not match its request count"), + const inputData = await readBatchInputs(args.inputFileContent); + const trace = await contextForCompletion(args.inputFileId, inputData); + if (!trace) { + logBatchInstrumentationError( + "left batch spans pending", + inputData.issues[0] ?? new Error("OpenAI Batch input does not match"), ); - } - if (issues.length > 0) { - logBatchInstrumentationError("left batch spans pending", issues[0]); return; } - - const task = await startBatchSpan(context); + const endTime = trace.endTime ?? getCurrentUnixTimestamp(); + const task = await startBatchSpan(trace.context); const taskParent = await task.export(); + const issues: Error[] = []; const seen = new Set(); await Promise.all([ completeResultFile({ - context, + context: trace.context, endTime, - file: args.outputFile, - inputs: inputData.inputs, + file: args.outputFileContent, + inputs: trace.inputs, issues, seen, source: "output", taskParent, }), completeResultFile({ - context, + context: trace.context, endTime, - file: args.errorFile, - inputs: inputData.inputs, + file: args.errorFileContent, + inputs: trace.inputs, issues, seen, source: "error", @@ -1076,40 +687,37 @@ async function completeBatch( return; } - const missing = [...inputData.inputs.values()].filter( + const missing = [...trace.inputs.values()].filter( ({ customId }) => !seen.has(customId), ); - if (status === "completed" && missing.length > 0) { - logBatchInstrumentationError( - "left batch spans pending", - new Error("OpenAI Batch result files are incomplete"), - ); - return; - } - for (const input of missing) { - try { - const child = await startBatchChild(context, taskParent, input); - child.log({ error: new Error(`OpenAI Batch ${status}`) }); + if (!trace.status || trace.status === "completed") { + if (missing.length > 0) { + logBatchInstrumentationError( + "left batch spans pending", + new Error("OpenAI Batch result files are incomplete"), + ); + return; + } + } else { + for (const input of missing) { + const child = await startBatchChild(trace.context, taskParent, input); + child.log({ error: new Error(`OpenAI Batch ${trace.status}`) }); child.end({ endTime }); - } catch (error) { - logBatchInstrumentationError("could not complete missing result", error); } - } - - if (status !== "completed") { - task.log({ error: new Error(`OpenAI Batch ${status}`) }); + task.log({ error: new Error(`OpenAI Batch ${trace.status}`) }); } task.end({ endTime }); + pendingBatchTraces.delete(args.inputFileId); } export const interceptOpenAIBatchTraceComplete: Parameters< typeof openAIChannels.batchesCompleteTrace.intercept >[0] = (target, thisArg, args) => - Promise.resolve(Reflect.apply(target, thisArg, args)).then(async (batch) => { + Promise.resolve(Reflect.apply(target, thisArg, args)).then(async (result) => { try { - await completeBatch({ ...args[0], batch }); + await completeBatch(args[0]); } catch (error) { logBatchInstrumentationError("could not complete batch", error); } - return batch; + return result; }); diff --git a/js/src/instrumentation/plugins/openai-channels.ts b/js/src/instrumentation/plugins/openai-channels.ts index 445a65a68..5ef873eca 100644 --- a/js/src/instrumentation/plugins/openai-channels.ts +++ b/js/src/instrumentation/plugins/openai-channels.ts @@ -19,9 +19,10 @@ import type { } from "../../vendor-sdk-types/openai"; import type { CompleteOpenAIBatchTraceArgs, - OpenAIBatchCreateParams, OpenAIBatchLike, - StartOpenAIBatchTraceArgs, + OpenAIBatchesRetrieveTraceArgs, + OpenAIFileLike, + OpenAIFilesCreateTraceArgs, } from "../../openai-batch-types"; type OpenAIChatSpanInfo = NonNullable["span_info"]>; @@ -37,19 +38,28 @@ type OpenAIResponsesChannelExtras = OpenAIChannelExtras; export const openAIChannels = defineChannels( "openai", { - batchesStartTrace: channel< - [StartOpenAIBatchTraceArgs], - OpenAIBatchCreateParams, + filesCreateTraced: channel< + [OpenAIFilesCreateTraceArgs], + OpenAIFileLike, OpenAIChannelExtras >({ - channelName: "batches.start-trace", + channelName: "files.create-traced", kind: "async", }), - batchesCompleteTrace: channel< - [CompleteOpenAIBatchTraceArgs], + batchesRetrieveTraced: channel< + [OpenAIBatchesRetrieveTraceArgs], OpenAIBatchLike, OpenAIChannelExtras + >({ + channelName: "batches.retrieve-traced", + kind: "async", + }), + + batchesCompleteTrace: channel< + [CompleteOpenAIBatchTraceArgs], + void, + OpenAIChannelExtras >({ channelName: "batches.complete-trace", kind: "async", diff --git a/js/src/instrumentation/plugins/openai-plugin.ts b/js/src/instrumentation/plugins/openai-plugin.ts index f20416e8c..a534764ae 100644 --- a/js/src/instrumentation/plugins/openai-plugin.ts +++ b/js/src/instrumentation/plugins/openai-plugin.ts @@ -15,8 +15,9 @@ import { processImagesInOutput, } from "./openai-span-data"; import { + interceptOpenAIBatchesRetrieveTraced, interceptOpenAIBatchTraceComplete, - interceptOpenAIBatchTraceStart, + interceptOpenAIFilesCreateTraced, } from "./openai-batch-instrumentation"; import { BRAINTRUST_CACHED_STREAM_METRIC, @@ -47,8 +48,11 @@ export class OpenAIPlugin extends BasePlugin { protected onEnable(): void { this.unsubscribers.push( - openAIChannels.batchesStartTrace.intercept( - interceptOpenAIBatchTraceStart, + openAIChannels.filesCreateTraced.intercept( + interceptOpenAIFilesCreateTraced, + ), + openAIChannels.batchesRetrieveTraced.intercept( + interceptOpenAIBatchesRetrieveTraced, ), openAIChannels.batchesCompleteTrace.intercept( interceptOpenAIBatchTraceComplete, diff --git a/js/src/openai-batch-types.ts b/js/src/openai-batch-types.ts index 0781f9b01..b6052d575 100644 --- a/js/src/openai-batch-types.ts +++ b/js/src/openai-batch-types.ts @@ -3,24 +3,28 @@ export type OpenAIBatchJSONL = | Iterable | AsyncIterable; -type OpenAIBatchFileContent = OpenAIBatchJSONL | Response; +export type OpenAIBatchFileContent = OpenAIBatchJSONL | Response; -type OpenAIBatchFile = +export type OpenAIBatchFile = | OpenAIBatchFileContent | PromiseLike; -type SupportedOpenAIBatchEndpoint = "/v1/chat/completions" | "/v1/responses"; +export interface OpenAIFileLike { + id: string; + [key: string]: unknown; +} export interface OpenAIBatchLike { id: string; endpoint: string; input_file_id: string; status: string; + created_at?: number | null; + in_progress_at?: number | null; completed_at?: number | null; failed_at?: number | null; expired_at?: number | null; cancelled_at?: number | null; - metadata?: Record | null; request_counts?: { completed?: number; failed?: number; @@ -29,47 +33,38 @@ export interface OpenAIBatchLike { [key: string]: unknown; } -export interface OpenAIBatchCreateParams { - input_file_id: string; - endpoint: SupportedOpenAIBatchEndpoint; - completion_window: "24h"; - metadata?: Record | null; - [key: string]: unknown; +export type OpenAIEnhancedResponse = { + response: Response; + data: T; + request_id?: string | null; +}; + +export interface OpenAIAPIPromise extends Promise { + withResponse(): Promise>; + asResponse(): Promise; } -export interface OpenAIBatchCreateInputParams { - endpoint: SupportedOpenAIBatchEndpoint; - completion_window: "24h"; - metadata?: Record | null; - [key: string]: unknown; +export interface OpenAIFilesResource { + create(params: TParams, options?: TOptions): OpenAIAPIPromise; } -export interface OpenAIFileLike { - id: string; - [key: string]: unknown; +export interface OpenAIBatchesResource { + retrieve(batchId: string, options?: TOptions): OpenAIAPIPromise; } -export interface OpenAIBatchWebhookEvent { - type: - | "batch.completed" - | "batch.failed" - | "batch.expired" - | "batch.cancelled"; - created_at?: number; - data: { id: string }; - [key: string]: unknown; +export interface OpenAIFilesCreateTraceArgs { + inputFileContent: OpenAIBatchFileContent; + parent: { export(): Promise } | { toStr(): string }; + params: unknown; } -export interface StartOpenAIBatchTraceArgs { - inputFile: OpenAIFileLike; - input: OpenAIBatchJSONL; - params: OpenAIBatchCreateInputParams; +export interface OpenAIBatchesRetrieveTraceArgs { + batchId: string; } -export interface CompleteOpenAIBatchTraceArgs { - batch: TBatch; - inputFile: OpenAIBatchFile; - outputFile?: OpenAIBatchFile; - errorFile?: OpenAIBatchFile; - webhook?: OpenAIBatchWebhookEvent; +export interface CompleteOpenAIBatchTraceArgs { + inputFileId: string; + inputFileContent: OpenAIBatchFile; + outputFileContent?: OpenAIBatchFile; + errorFileContent?: OpenAIBatchFile; } diff --git a/js/src/openai-batch.ts b/js/src/openai-batch.ts index b0b4872eb..fa0837684 100644 --- a/js/src/openai-batch.ts +++ b/js/src/openai-batch.ts @@ -1,65 +1,285 @@ import { openAIChannels } from "./instrumentation/plugins/openai-channels"; +import { getSpanParentObject } from "./logger"; +import { + createLazyAPIPromise, + type EnhancedResponse, +} from "./wrappers/openai-promise-utils"; import type { CompleteOpenAIBatchTraceArgs, - OpenAIBatchCreateParams, + OpenAIAPIPromise, OpenAIBatchLike, - StartOpenAIBatchTraceArgs, + OpenAIBatchesResource, + OpenAIBatchFileContent, + OpenAIFileLike, + OpenAIFilesResource, } from "./openai-batch-types"; +function read(value: unknown, key: PropertyKey): unknown { + if ( + value === null || + (typeof value !== "object" && typeof value !== "function") + ) { + return undefined; + } + try { + return Reflect.get(value, key); + } catch { + return undefined; + } +} + +function isAsyncIterable(value: unknown): value is AsyncIterable { + return typeof read(value, Symbol.asyncIterator) === "function"; +} + +function teeAsyncIterable(source: AsyncIterable): { + upload: AsyncIterable; + trace: Response; +} { + const iterator = source[Symbol.asyncIterator](); + const stream = new ReadableStream({ + async pull(controller) { + try { + const result = await iterator.next(); + if (result.done) { + controller.close(); + return; + } + if (!(result.value instanceof Uint8Array)) { + throw new TypeError( + "OpenAI upload streams must yield Uint8Array chunks", + ); + } + controller.enqueue(result.value); + } catch (error) { + controller.error(error); + } + }, + async cancel(reason) { + await iterator.return?.(reason); + }, + }); + const [uploadStream, trace] = stream.tee(); + const upload = { + async *[Symbol.asyncIterator]() { + const reader = uploadStream.getReader(); + try { + while (true) { + const result = await reader.read(); + if (result.done) { + return; + } + yield result.value; + } + } finally { + reader.releaseLock(); + } + }, + }; + const path = read(source, "path"); + if (typeof path === "string") { + Object.defineProperty(upload, "path", { value: path }); + } + return { upload, trace: new Response(trace) }; +} + +function teeUpload( + upload: unknown, +): { upload: unknown; trace: OpenAIBatchFileContent } | undefined { + if (upload instanceof Blob) { + return { upload, trace: new Response(upload) }; + } + if (upload instanceof Response) { + return { upload, trace: upload.clone() }; + } + if (isAsyncIterable(upload)) { + return teeAsyncIterable(upload); + } + return undefined; +} + /** - * Start a trace for a new OpenAI Batch, with pending spans for its requests, - * and return the parameters to pass to `client.batches.create()`. + * Wrap `openai.files.create` to trace an OpenAI Batch input file upload. + * + * Pass the `openai.files` resource, then call the returned function with the + * same arguments as `openai.files.create`. Files whose purpose is not `batch` + * are passed through without tracing. Blob and Response uploads are copied for + * tracing; async-iterable uploads (including Node file streams) are teed so the + * exact bytes still flow to OpenAI while Braintrust reads a separate branch. + * + * The returned input file ID identifies the trace. Each request span is + * identified by that file ID and its JSONL `custom_id`, so use every traced + * input file for only one OpenAI Batch. * - * Supported endpoints are `/v1/chat/completions` and `/v1/responses`. + * @example + * ```ts + * const inputFile = await openaiFilesCreateTraced(openai.files)({ + * file: createReadStream("batch.jsonl"), + * purpose: "batch", + * }); + * ``` */ -export async function startOpenAIBatchTrace( - args: StartOpenAIBatchTraceArgs, -): Promise { - return await openAIChannels.batchesStartTrace.invoke( - async (input) => ({ - ...input.params, - input_file_id: input.inputFile.id, - }), - undefined, - [args], - {}, - ); +export function openaiFilesCreateTraced< + TParams, + TFile extends OpenAIFileLike, + TOptions = unknown, +>( + files: OpenAIFilesResource, +): (params: TParams, options?: TOptions) => OpenAIAPIPromise { + return (params, options) => { + const parent = getSpanParentObject(); + const purpose = read(params, "purpose"); + const file = read(params, "file"); + if (purpose !== "batch") { + return files.create(params, options); + } + + let branches: ReturnType; + try { + branches = teeUpload(file); + } catch { + return files.create(params, options); + } + if (!branches) { + return files.create(params, options); + } + + const tracedParams = { + ...Object(params), + file: branches.upload, + } as TParams; + const apiPromise = files.create(tracedParams, options); + let enhancedResponse: EnhancedResponse | undefined; + const tracedResponse = openAIChannels.filesCreateTraced + .invoke( + async () => { + enhancedResponse = await apiPromise.withResponse(); + return enhancedResponse.data; + }, + files, + [{ params, inputFileContent: branches.trace, parent }], + {}, + ) + .then((data) => { + if (!enhancedResponse) { + throw new Error( + "Expected OpenAI withResponse() to provide a response", + ); + } + return { ...enhancedResponse, data }; + }); + + return createLazyAPIPromise( + () => tracedResponse, + () => tracedResponse.then(({ response }) => response), + () => apiPromise, + ); + }; +} + +/** + * Wrap `openai.batches.retrieve` to update an OpenAI Batch trace with the + * server-recorded lifecycle timestamps and close its spans at a terminal + * status. + * + * Pass the `openai.batches` resource, then call the returned function with the + * same arguments as `openai.batches.retrieve`. The Batch must use an input file + * previously uploaded through `openaiFilesCreateTraced` in this process. + * + * @example + * ```ts + * const batch = await openaiBatchesRetrieveTraced(openai.batches)(batchId); + * ``` + */ +export function openaiBatchesRetrieveTraced< + TBatch extends OpenAIBatchLike, + TOptions = unknown, +>( + batches: OpenAIBatchesResource, +): (batchId: string, options?: TOptions) => OpenAIAPIPromise { + return (batchId, options) => { + const apiPromise = batches.retrieve(batchId, options); + let enhancedResponse: EnhancedResponse | undefined; + const tracedResponse = openAIChannels.batchesRetrieveTraced + .invoke( + async () => { + enhancedResponse = await apiPromise.withResponse(); + return enhancedResponse.data; + }, + batches, + [{ batchId }], + {}, + ) + .then((data) => { + if (!enhancedResponse) { + throw new Error( + "Expected OpenAI withResponse() to provide a response", + ); + } + return { ...enhancedResponse, data }; + }); + + return createLazyAPIPromise( + () => tracedResponse, + () => tracedResponse.then(({ response }) => response), + () => apiPromise, + ); + }; } /** - * Complete an OpenAI Batch trace from caller-supplied Batch API data. + * Add result data to spans created by `openaiFilesCreateTraced`. + * + * This helper performs no OpenAI API requests. Pass the input file ID and the + * input, output, and error contents returned by `openai.files.content`, or pass + * JSONL strings/iterables directly. Response bodies are consumed. Call + * `openaiBatchesRetrieveTraced` first when exact OpenAI lifecycle timestamps + * are available. * - * This helper performs no OpenAI API requests. Pass the Responses or promises - * returned by `client.files.content()` for the input and result files, JSONL - * strings, or iterables of parsed records. Supplied Response bodies are - * consumed directly. + * @example + * ```ts + * const outputFileContent = batch.output_file_id + * ? await openai.files.content(batch.output_file_id) + * : undefined; + * const errorFileContent = batch.error_file_id + * ? await openai.files.content(batch.error_file_id) + * : undefined; + * await completeOpenAIBatchTrace({ + * inputFileId: batch.input_file_id, + * inputFileContent: await openai.files.content(batch.input_file_id), + * outputFileContent, + * errorFileContent, + * }); + * ``` */ -export async function completeOpenAIBatchTrace( - args: CompleteOpenAIBatchTraceArgs, -): Promise { - const inputFile = Promise.resolve(args.inputFile); - const outputFile = - args.outputFile === undefined +export async function completeOpenAIBatchTrace( + args: CompleteOpenAIBatchTraceArgs, +): Promise { + const inputFileContent = Promise.resolve(args.inputFileContent); + const outputFileContent = + args.outputFileContent === undefined + ? undefined + : Promise.resolve(args.outputFileContent); + const errorFileContent = + args.errorFileContent === undefined ? undefined - : Promise.resolve(args.outputFile); - const errorFile = - args.errorFile === undefined ? undefined : Promise.resolve(args.errorFile); + : Promise.resolve(args.errorFileContent); - // Content requests are commonly started before this helper is called. Attach - // rejection handlers immediately so early no-op paths and disabled - // instrumentation cannot leave caller-supplied promises unhandled. - void inputFile.catch(() => undefined); - void outputFile?.catch(() => undefined); - void errorFile?.catch(() => undefined); + void inputFileContent.catch(() => undefined); + void outputFileContent?.catch(() => undefined); + void errorFileContent?.catch(() => undefined); - const batch = await openAIChannels.batchesCompleteTrace.invoke( - async (input) => input.batch, + await openAIChannels.batchesCompleteTrace.invoke( + async () => undefined, undefined, - [{ ...args, inputFile, outputFile, errorFile }], + [ + { + ...args, + inputFileContent, + outputFileContent, + errorFileContent, + }, + ], {}, ); - // The completion channel preserves the exact batch object supplied by the - // caller, including fields from a newer OpenAI SDK version. - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return batch as TBatch; }