From 45f78d65ea1cd3ad317cc6ff2296dcb6e79bca3c Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Thu, 13 Aug 2026 22:49:35 -0600 Subject: [PATCH 1/4] bump spec version --- .../com/google/genai/BraintrustApiClient.java | 2 +- .../genai/v1_18_0/BraintrustGenAITest.java | 4 ++-- .../sdkspecimpl/SpecClientRegistry.java | 17 +++++++++++++++- .../sdkspecimpl/clients/GoogleSpecClient.java | 20 ++++++++++++++++++- gradle.properties | 2 +- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java b/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java index dc52db2d..1a108cdb 100644 --- a/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java +++ b/braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java @@ -55,7 +55,7 @@ private void tagSpan( @Nullable String responseBody) { try { Map metadata = new java.util.HashMap<>(); - metadata.put("provider", "gemini"); + metadata.put("provider", "google"); // Parse request if (requestBody != null) { diff --git a/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/dev/braintrust/instrumentation/genai/v1_18_0/BraintrustGenAITest.java b/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/dev/braintrust/instrumentation/genai/v1_18_0/BraintrustGenAITest.java index 35b2c683..bc2e066e 100644 --- a/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/dev/braintrust/instrumentation/genai/v1_18_0/BraintrustGenAITest.java +++ b/braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/dev/braintrust/instrumentation/genai/v1_18_0/BraintrustGenAITest.java @@ -67,7 +67,7 @@ void testWrapGemini() { span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata")); assertNotNull(metadataJson, "braintrust.metadata should be set"); var metadata = JSON_MAPPER.readTree(metadataJson); - assertEquals("gemini", metadata.get("provider").asText()); + assertEquals("google", metadata.get("provider").asText()); assertEquals(MODEL_ID, metadata.get("model").asText()); assertEquals(0.0, metadata.get("temperature").asDouble()); assertEquals(50, metadata.get("maxOutputTokens").asInt()); @@ -145,7 +145,7 @@ void testWrapGeminiAsync() { span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata")); assertNotNull(metadataJson, "braintrust.metadata should be set"); var metadata = JSON_MAPPER.readTree(metadataJson); - assertEquals("gemini", metadata.get("provider").asText()); + assertEquals("google", metadata.get("provider").asText()); assertEquals(MODEL_ID, metadata.get("model").asText()); assertEquals(0.0, metadata.get("temperature").asDouble()); assertEquals(50, metadata.get("maxOutputTokens").asInt()); diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java index ad3987e9..e5eebeb1 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java @@ -32,7 +32,22 @@ public final class SpecClientRegistry { * nothing instead of producing a failing "unsupported" test. Prefer registering a client; reach * for this only when no Java client can express the spec at all. */ - private static final Set KNOWN_UNSUPPORTED_SPECS = Set.of(); + private static final Set KNOWN_UNSUPPORTED_SPECS = + Set.of( + // Brand-new Google features added in braintrust-spec v0.0.10. The genai + // instrumentation / GoogleSpecClient don't implement them yet (thinking, + // grounding tools, response modalities, per-modality prompt-token extraction), + // so they're marked unsupported rather than half-implemented. GoogleSpecClient + // also filters these out of supports(); see its UNSUPPORTED_SPECS. + "google/thinking", + "google/grounding", + "google/streaming", + "google/generated_audio_usage", + "google/generated_image_usage", + "google/attachments", + // Google Interactions API (/v1/interactions) — no Java client exists. + "google/interactions", + "google/interactions_streaming"); private static final List CLIENTS = Stream.of( diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java index a2bb597e..8ff00341 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/GoogleSpecClient.java @@ -17,6 +17,23 @@ /** Google Gemini (google-genai) client: generateContent (sync + streaming). */ public final class GoogleSpecClient implements SpecClient { + /** + * {@code :generateContent} specs that require Google features not yet implemented in the genai + * instrumentation / this client (thinking, grounding tools, response modalities, and + * per-modality prompt-token extraction for attachments). They are additionally listed in {@link + * dev.braintrust.sdkspecimpl.SpecClientRegistry#KNOWN_UNSUPPORTED_SPECS} so they load to a + * deliberate skip instead of a failing "unsupported" sentinel. Remove an entry here once the + * corresponding feature lands. + */ + private static final java.util.Set UNSUPPORTED_SPECS = + java.util.Set.of( + "thinking", + "grounding", + "streaming", + "generated_audio_usage", + "generated_image_usage", + "attachments"); + private volatile Client geminiClient; @Override @@ -31,7 +48,8 @@ public String provider() { @Override public boolean supports(LlmSpanSpec spec) { - return spec.endpoint().contains(":generateContent"); + return spec.endpoint().contains(":generateContent") + && !UNSUPPORTED_SPECS.contains(spec.name()); } @Override diff --git a/gradle.properties b/gradle.properties index 103fc769..52a608c5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,7 +8,7 @@ org.gradle.daemon=true org.gradle.warning.mode=summary # braintrust-spec git ref (SHA or tag) used by btx tests -braintrustSpecRef=v0.0.9 +braintrustSpecRef=v0.0.10 # braintrust-openapi commit SHA used by braintrust-api braintrustOpenApiRef=64b79cb9122f50a74eac98ea86c3ec1858c0cdd1 From 5f077342d557ffad9e19b5612840c90f8ff6eaf5 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 17 Aug 2026 16:18:28 -0600 Subject: [PATCH 2/4] btx: support nested spans in spec yaml assertion --- .../sdkspecimpl/LlmSpanSpecTest.java | 25 +++++++++-- .../braintrust/sdkspecimpl/SpanConverter.java | 8 ++++ .../braintrust/sdkspecimpl/SpanFetcher.java | 45 ++++++++++++++++++- .../braintrust/sdkspecimpl/SpanValidator.java | 3 ++ 4 files changed, 76 insertions(+), 5 deletions(-) diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/LlmSpanSpecTest.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/LlmSpanSpecTest.java index bd56b1f7..9f939cd9 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/LlmSpanSpecTest.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/LlmSpanSpecTest.java @@ -90,8 +90,9 @@ static Stream specs() throws Exception { SpecClientRegistry.execute( spec, CTX); totalExpectedSpans.addAndGet( - spec.expectedBrainstoreSpans() - .size() + countExpectedSpans( + spec + .expectedBrainstoreSpans()) + 1); return Arguments.of( spec, rootSpanId); @@ -111,9 +112,27 @@ void runSpec(LlmSpanSpec spec, String rootSpanId) throws Exception { if (SpecClientRegistry.UNSUPPORTED_CLIENT_ID.equals(spec.client())) { org.junit.jupiter.api.Assertions.fail(SpecClientRegistry.unsupportedSpecMessage(spec)); } - int expectedSpanCount = spec.expectedBrainstoreSpans().size(); + int expectedSpanCount = countExpectedSpans(spec.expectedBrainstoreSpans()); List> brainstoreSpans = SPAN_FETCHER.fetch(rootSpanId, expectedSpanCount); SpanValidator.validate(brainstoreSpans, spec.expectedBrainstoreSpans(), spec.displayName()); } + + /** + * Total number of spans a spec expects, counting nested {@code child_spans} recursively. Used + * to size the {@code awaitExportedSpans} gate so it doesn't return before child (e.g. tool) + * spans have flushed. + */ + @SuppressWarnings("unchecked") + private static int countExpectedSpans(List> spans) { + int total = 0; + for (Map span : spans) { + total += 1; + Object children = span.get("child_spans"); + if (children instanceof List l) { + total += countExpectedSpans((List>) l); + } + } + return total; + } } diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanConverter.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanConverter.java index 49ee945e..890e37c0 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanConverter.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanConverter.java @@ -57,6 +57,14 @@ private static boolean isLlmInstrumentationSpan(SpanData span) { private static Map toSingleBrainstoreSpan(SpanData span) { Map result = new LinkedHashMap<>(); + // span_id / span_parents mirror the brainstore fields so the fetch layer can rebuild the + // parent→child tree in REPLAY mode (live BTQL spans already carry these). + result.put("span_id", span.getSpanContext().getSpanId()); + var parentContext = span.getParentSpanContext(); + result.put( + "span_parents", + parentContext.isValid() ? List.of(parentContext.getSpanId()) : null); + result.put("name", span.getName()); result.put("metrics", parseJsonMap(span, "braintrust.metrics")); result.put("metadata", parseJsonMap(span, "braintrust.metadata")); diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanFetcher.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanFetcher.java index fbd69d9c..29b47aa1 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanFetcher.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanFetcher.java @@ -41,7 +41,7 @@ public List> fetch(String rootSpanId, int numExpectedChildSp if (isReplayMode()) { // Fast path: convert the in-memory OTel spans to brainstore format locally. - return convertedOtelSpans; + return buildSpanTree(convertedOtelSpans); } // Live path: spans were actually sent to Braintrust — fetch them back via BTQL. @@ -86,7 +86,45 @@ public List> fetch(String rootSpanId, int numExpectedChildSp // Cross-check that our local OTel→brainstore conversion matches the real thing. assertConverterMatchesBrainstore(convertedOtelSpans, brainstoreSpans, rootSpanId); - return brainstoreSpans; + return buildSpanTree(brainstoreSpans); + } + + /** + * Reshape a flat list of brainstore spans into a forest: each span gets a nested {@code + * child_spans} list, and only top-level spans (those whose parent is not among the fetched + * spans — i.e. children of the already-excluded root wrapper) are returned. Sibling order is + * preserved from the input list. This lets specs assert nested tool spans (e.g. a {@code + * web_search_call} child of an LLM span) via a recursive {@code child_spans:} structure. + */ + @SuppressWarnings("unchecked") + static List> buildSpanTree(List> flatSpans) { + // Work on mutable copies so we can attach child_spans without mutating the inputs. + List> spans = new java.util.ArrayList<>(); + Map> byId = new java.util.LinkedHashMap<>(); + for (Map span : flatSpans) { + Map copy = new java.util.LinkedHashMap<>(span); + copy.put("child_spans", new java.util.ArrayList>()); + spans.add(copy); + Object id = span.get("span_id"); + if (id instanceof String s) { + byId.put(s, copy); + } + } + + List> topLevel = new java.util.ArrayList<>(); + for (Map span : spans) { + Map parent = null; + Object parents = span.get("span_parents"); + if (parents instanceof List l && !l.isEmpty() && l.get(0) instanceof String pid) { + parent = byId.get(pid); + } + if (parent != null) { + ((List>) parent.get("child_spans")).add(span); + } else { + topLevel.add(span); + } + } + return topLevel; } /** @@ -131,6 +169,9 @@ private static void assertConverterMatchesBrainstore( Map convWithoutName = new java.util.LinkedHashMap<>(conv); convWithoutName.remove("name"); convWithoutName.remove("metrics"); + // span_id / span_parents are structural (used only for tree-building), not content. + convWithoutName.remove("span_id"); + convWithoutName.remove("span_parents"); assertIsSubset(convWithoutName, realSpan, ctx); assertMetricsKeysPresent(conv, realSpan, ctx); } diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java index 4029fbdf..387ed177 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java @@ -18,6 +18,9 @@ *
  • {@code span_attributes} — type, name *
  • {@code input} — input messages *
  • {@code output} — output choices / content + *
  • {@code child_spans} — a nested list of child-span assertions, validated recursively against + * the span's actual children (built by {@link SpanFetcher#buildSpanTree}). Handled by the + * generic recursion below; a span that omits it asserts nothing about its children. * * *

    Spans arrive here already in brainstore format, produced either by {@link SpanConverter} From 1a8231b26bd8125d1beb76931df42ba2d4a1d286 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 17 Aug 2026 22:18:12 -0600 Subject: [PATCH 3/4] wip -- web search --- .../anthropic/v2_2_0/TracingHttpClient.java | 55 +- .../BraintrustAnthropicWebSearchTest.java | 135 +++++ .../langchain/v1_8_0/WrappedHttpClient.java | 31 +- .../openai/v2_15_0/TracingHttpClient.java | 55 +- .../openai/v2_15_0/BraintrustOpenAITest.java | 2 + .../BraintrustOpenAIWebSearchTest.java | 143 +++++ .../springai/v1_0_0/BraintrustSpringAI.java | 23 +- .../InstrumentationSemConv.java | 502 ++++++++++++++++++ .../InstrumentationSemConvChildSpansTest.java | 355 +++++++++++++ .../sdkspecimpl/SpecClientRegistry.java | 15 +- .../clients/SpringAi1AnthropicSpecClient.java | 13 +- 11 files changed, 1288 insertions(+), 41 deletions(-) create mode 100644 braintrust-sdk/instrumentation/anthropic_2_2_0/src/test/java/dev/braintrust/instrumentation/anthropic/v2_2_0/BraintrustAnthropicWebSearchTest.java create mode 100644 braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAIWebSearchTest.java create mode 100644 braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvChildSpansTest.java diff --git a/braintrust-sdk/instrumentation/anthropic_2_2_0/src/main/java/dev/braintrust/instrumentation/anthropic/v2_2_0/TracingHttpClient.java b/braintrust-sdk/instrumentation/anthropic_2_2_0/src/main/java/dev/braintrust/instrumentation/anthropic/v2_2_0/TracingHttpClient.java index 207196b5..572e68dc 100644 --- a/braintrust-sdk/instrumentation/anthropic_2_2_0/src/main/java/dev/braintrust/instrumentation/anthropic/v2_2_0/TracingHttpClient.java +++ b/braintrust-sdk/instrumentation/anthropic_2_2_0/src/main/java/dev/braintrust/instrumentation/anthropic/v2_2_0/TracingHttpClient.java @@ -24,6 +24,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -51,10 +52,11 @@ public TracingHttpClient(OpenTelemetry openTelemetry, HttpClient underlying) { * instrumented — a long-lived client wrapped inside some unrelated span would otherwise parent * every future request to that stale span. */ - private Span startLlmSpan(@Nullable Context headerContext) { + private Span startLlmSpan(@Nullable Context headerContext, Instant startTime) { Context parent = headerContext != null ? headerContext : Context.current(); return tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME) .setParent(parent) + .setStartTimestamp(startTime) .startSpan(); } @@ -110,7 +112,8 @@ public void close() { public @Nonnull HttpResponse execute( @Nonnull HttpRequest httpRequest, @Nonnull RequestOptions requestOptions) { var extracted = extractCallerContext(httpRequest); - var span = startLlmSpan(extracted.callerContext()); + var llmSpanStart = Instant.now(); + var span = startLlmSpan(extracted.callerContext(), llmSpanStart); try (var ignored = span.makeCurrent()) { var bufferedRequest = bufferRequestBody(extracted.request()); @@ -128,7 +131,7 @@ public void close() { inputJson); var response = underlying.execute(bufferedRequest, requestOptions); - return new TeeingStreamHttpResponse(response, span); + return new TeeingStreamHttpResponse(response, span, tracer, llmSpanStart); } catch (Exception e) { InstrumentationSemConv.tagLLMSpanResponse(span, e); span.end(); @@ -140,7 +143,8 @@ public void close() { public @Nonnull CompletableFuture executeAsync( @Nonnull HttpRequest httpRequest, @Nonnull RequestOptions requestOptions) { var extracted = extractCallerContext(httpRequest); - var span = startLlmSpan(extracted.callerContext()); + var llmSpanStart = Instant.now(); + var span = startLlmSpan(extracted.callerContext(), llmSpanStart); try { var bufferedRequest = bufferRequestBody(extracted.request()); String inputJson = @@ -157,7 +161,10 @@ public void close() { return underlying .executeAsync(bufferedRequest, requestOptions) .thenApply( - response -> (HttpResponse) new TeeingStreamHttpResponse(response, span)) + response -> + (HttpResponse) + new TeeingStreamHttpResponse( + response, span, tracer, llmSpanStart)) .whenComplete( (response, t) -> { if (t != null) { @@ -237,14 +244,19 @@ private static String readBodyAsString(HttpRequestBody body) { private static final class TeeingStreamHttpResponse implements HttpResponse { private final HttpResponse delegate; private final Span span; + private final Tracer tracer; + private final Instant llmSpanStart; private final long spanStartNanos = System.nanoTime(); private final AtomicLong timeToFirstTokenNanos = new AtomicLong(); private final ByteArrayOutputStream teeBuffer = new ByteArrayOutputStream(); private final InputStream teeStream; - TeeingStreamHttpResponse(HttpResponse delegate, Span span) { + TeeingStreamHttpResponse( + HttpResponse delegate, Span span, Tracer tracer, Instant llmSpanStart) { this.delegate = delegate; this.span = span; + this.tracer = tracer; + this.llmSpanStart = llmSpanStart; this.teeStream = new TeeInputStream( delegate.body(), teeBuffer, this::onFirstByte, this::onStreamClosed); @@ -260,7 +272,18 @@ private void onStreamClosed() { synchronized (teeBuffer) { bytes = teeBuffer.toByteArray(); } - tagSpanFromBuffer(span, bytes, timeToFirstTokenNanos.get()); + String responseJson = tagSpanFromBuffer(span, bytes, timeToFirstTokenNanos.get()); + if (responseJson != null) { + // Emit child spans for server-side tool calls (web search, etc.) nested under + // the LLM span, while it is still live. Anchored at the LLM span start with + // zero duration — providers don't report per-tool timing. + InstrumentationSemConv.addServerSideChildSpans( + tracer, + span, + InstrumentationSemConv.PROVIDER_NAME_ANTHROPIC, + responseJson, + llmSpanStart); + } } finally { span.end(); } @@ -354,8 +377,8 @@ private void notifyClosed() { // Span tagging from buffered bytes // ------------------------------------------------------------------------- - private static void tagSpanFromBuffer(Span span, byte[] bytes, Long timeToFirstTokenNanos) { - if (bytes.length == 0) return; + private static String tagSpanFromBuffer(Span span, byte[] bytes, Long timeToFirstTokenNanos) { + if (bytes.length == 0) return null; try { String firstLine = firstNonEmptyLine(bytes); // Anthropic SSE starts with "event: message_start\ndata: ..." so we detect @@ -364,17 +387,17 @@ private static void tagSpanFromBuffer(Span span, byte[] bytes, Long timeToFirstT firstLine != null && (firstLine.startsWith("data:") || firstLine.startsWith("event:")); if (isSse) { - tagSpanFromSseBytes(span, bytes, timeToFirstTokenNanos); + return tagSpanFromSseBytes(span, bytes, timeToFirstTokenNanos); } else { // Non-streaming: plain Message JSON — pass it whole, no time_to_first_token + String responseJson = new String(bytes, StandardCharsets.UTF_8); InstrumentationSemConv.tagLLMSpanResponse( - span, - InstrumentationSemConv.PROVIDER_NAME_ANTHROPIC, - new String(bytes, StandardCharsets.UTF_8), - null); + span, InstrumentationSemConv.PROVIDER_NAME_ANTHROPIC, responseJson, null); + return responseJson; } } catch (Exception e) { log.error("Could not tag span from Anthropic response buffer", e); + return null; } } @@ -405,7 +428,7 @@ private static String firstNonEmptyLine(byte[] bytes) { * field inside the JSON. Feed each data payload to {@link MessageAccumulator} and serialize the * assembled {@link com.anthropic.models.messages.Message} for the span. */ - private static void tagSpanFromSseBytes( + private static String tagSpanFromSseBytes( Span span, byte[] sseBytes, Long timeToFirstTokenNanos) { try { var mapper = BraintrustJsonMapper.get(); @@ -431,8 +454,10 @@ private static void tagSpanFromSseBytes( InstrumentationSemConv.PROVIDER_NAME_ANTHROPIC, assembledMessageJson, timeToFirstTokenNanos); + return assembledMessageJson; } catch (Exception e) { log.error("Could not parse Anthropic SSE buffer to tag streaming span output", e); + return null; } } } diff --git a/braintrust-sdk/instrumentation/anthropic_2_2_0/src/test/java/dev/braintrust/instrumentation/anthropic/v2_2_0/BraintrustAnthropicWebSearchTest.java b/braintrust-sdk/instrumentation/anthropic_2_2_0/src/test/java/dev/braintrust/instrumentation/anthropic/v2_2_0/BraintrustAnthropicWebSearchTest.java new file mode 100644 index 00000000..d2446532 --- /dev/null +++ b/braintrust-sdk/instrumentation/anthropic_2_2_0/src/test/java/dev/braintrust/instrumentation/anthropic/v2_2_0/BraintrustAnthropicWebSearchTest.java @@ -0,0 +1,135 @@ +package dev.braintrust.instrumentation.anthropic.v2_2_0; + +import static org.junit.jupiter.api.Assertions.*; + +import com.anthropic.client.AnthropicClient; +import com.anthropic.client.okhttp.AnthropicOkHttpClient; +import com.anthropic.models.messages.MessageCreateParams; +import com.anthropic.models.messages.Model; +import com.anthropic.models.messages.WebSearchTool20250305; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.TestHarness; +import dev.braintrust.instrumentation.Instrumenter; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.trace.data.SpanData; +import java.util.List; +import lombok.SneakyThrows; +import net.bytebuddy.agent.ByteBuddyAgent; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Verifies that Anthropic server-side web search is captured both as a cost metric on the LLM span + * ({@code server_tool_use_web_search_requests}) and as a child {@code type:"tool"} span parented to + * the LLM span, giving web search its own cost/latency visibility on the trace timeline. + */ +public class BraintrustAnthropicWebSearchTest { + private static final String TEST_MODEL = "claude-sonnet-4-5-20250929"; + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + private static final AttributeKey SPAN_ATTRIBUTES = + AttributeKey.stringKey("braintrust.span_attributes"); + private static final AttributeKey METADATA = + AttributeKey.stringKey("braintrust.metadata"); + private static final AttributeKey METRICS = + AttributeKey.stringKey("braintrust.metrics"); + + @BeforeAll + public static void beforeAll() { + var instrumentation = ByteBuddyAgent.install(); + Instrumenter.install( + instrumentation, BraintrustAnthropicWebSearchTest.class.getClassLoader()); + } + + private TestHarness testHarness; + + @BeforeEach + void beforeEach() { + testHarness = TestHarness.setup(); + } + + private static MessageCreateParams webSearchRequest() { + return MessageCreateParams.builder() + .model(Model.of(TEST_MODEL)) + .maxTokens(1024) + .addUserMessage( + "Search the web for one recent AI news headline and answer in one" + + " sentence.") + .addTool(WebSearchTool20250305.builder().maxUses(3).build()) + .build(); + } + + @Test + @SneakyThrows + void testWebSearch() { + AnthropicClient client = + AnthropicOkHttpClient.builder() + .baseUrl(testHarness.anthropicBaseUrl()) + .apiKey(testHarness.anthropicApiKey()) + .build(); + + var response = client.messages().create(webSearchRequest()); + assertNotNull(response); + + assertWebSearch(testHarness.awaitExportedSpans(2)); + } + + @Test + @SneakyThrows + void testWebSearchStreaming() { + AnthropicClient client = + AnthropicOkHttpClient.builder() + .baseUrl(testHarness.anthropicBaseUrl()) + .apiKey(testHarness.anthropicApiKey()) + .build(); + + try (var stream = client.messages().createStreaming(webSearchRequest())) { + stream.stream().forEach(event -> {}); + } + + assertWebSearch(testHarness.awaitExportedSpans(2)); + } + + @SneakyThrows + private static void assertWebSearch(List spans) { + var llmSpans = spans.stream().filter(s -> isType(s, "llm")).toList(); + assertEquals(1, llmSpans.size(), "expected a single LLM span"); + var llm = llmSpans.get(0); + + // Cost metric on the LLM span. + JsonNode metrics = JSON_MAPPER.readTree(llm.getAttributes().get(METRICS)); + assertTrue( + metrics.has("server_tool_use_web_search_requests"), + "expected server_tool_use_web_search_requests metric, got: " + metrics); + assertTrue(metrics.get("server_tool_use_web_search_requests").asDouble() >= 1.0); + + // At least one web_search tool span, parented to the LLM span. + var toolSpans = + spans.stream() + .filter(s -> isType(s, "tool")) + .filter(s -> "web_search".equals(s.getName())) + .toList(); + assertFalse( + toolSpans.isEmpty(), + "expected at least one web_search tool span, got: " + + spans.stream().map(SpanData::getName).toList()); + + for (var tool : toolSpans) { + assertEquals( + llm.getSpanId(), + tool.getParentSpanId(), + "web_search tool span must be a child of the LLM span"); + JsonNode metadata = JSON_MAPPER.readTree(tool.getAttributes().get(METADATA)); + assertEquals("server_tool_use", metadata.path("tool_call_type").asText()); + assertEquals("web_search_tool_result", metadata.path("tool_result_type").asText()); + assertFalse(metadata.path("tool_use_id").asText().isEmpty()); + } + } + + @SneakyThrows + private static boolean isType(SpanData span, String type) { + String attr = span.getAttributes().get(SPAN_ATTRIBUTES); + return attr != null && type.equals(JSON_MAPPER.readTree(attr).path("type").asText()); + } +} diff --git a/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/WrappedHttpClient.java b/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/WrappedHttpClient.java index cfeeec89..adc69162 100644 --- a/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/WrappedHttpClient.java +++ b/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/WrappedHttpClient.java @@ -18,6 +18,7 @@ import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; import java.net.URI; +import java.time.Instant; import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicLong; @@ -41,15 +42,19 @@ public WrappedHttpClient( @Override public SuccessfulHttpResponse execute(HttpRequest request) throws HttpException, RuntimeException { + Instant llmSpanStart = Instant.now(); Span span = tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME) .setSpanKind(SpanKind.CLIENT) + .setStartTimestamp(llmSpanStart) .startSpan(); try (Scope scope = span.makeCurrent()) { tagRequest(span, request); var response = underlying.execute(request); InstrumentationSemConv.tagLLMSpanResponse( span, options.providerName(), response.body()); + InstrumentationSemConv.addServerSideChildSpans( + tracer, span, options.providerName(), response.body(), llmSpanStart); return response; } catch (Throwable t) { InstrumentationSemConv.tagLLMSpanResponse(span, t); @@ -65,15 +70,18 @@ public void execute(HttpRequest request, ServerSentEventListener listener) { underlying.execute(request, listener); return; } + Instant llmSpanStart = Instant.now(); Span span = tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME) .setSpanKind(SpanKind.CLIENT) + .setStartTimestamp(llmSpanStart) .startSpan(); try (Scope ignored = span.makeCurrent()) { tagRequest(span, request); underlying.execute( request, - new WrappedServerSentEventListener(listener, span, options.providerName())); + new WrappedServerSentEventListener( + listener, span, options.providerName(), tracer, llmSpanStart)); } catch (Throwable t) { InstrumentationSemConv.tagLLMSpanResponse(span, t); span.end(); @@ -88,16 +96,19 @@ public void execute( underlying.execute(request, parser, listener); return; } + Instant llmSpanStart = Instant.now(); Span span = tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME) .setSpanKind(SpanKind.CLIENT) + .setStartTimestamp(llmSpanStart) .startSpan(); try (Scope ignored = span.makeCurrent()) { tagRequest(span, request); underlying.execute( request, parser, - new WrappedServerSentEventListener(listener, span, options.providerName())); + new WrappedServerSentEventListener( + listener, span, options.providerName(), tracer, llmSpanStart)); } catch (Throwable t) { InstrumentationSemConv.tagLLMSpanResponse(span, t); span.end(); @@ -122,16 +133,24 @@ static class WrappedServerSentEventListener implements ServerSentEventListener { private final ServerSentEventListener delegate; private final Span span; private final String providerName; + private final Tracer tracer; + private final Instant llmSpanStart; private final long startNanos = System.nanoTime(); private final AtomicLong timeToFirstTokenNanos = new AtomicLong(); private final SseResponseAccumulator accumulator = new SseResponseAccumulator(BraintrustJsonMapper.get()); WrappedServerSentEventListener( - ServerSentEventListener delegate, Span span, String providerName) { + ServerSentEventListener delegate, + Span span, + String providerName, + Tracer tracer, + Instant llmSpanStart) { this.delegate = delegate; this.span = span; this.providerName = providerName; + this.tracer = tracer; + this.llmSpanStart = llmSpanStart; } @Override @@ -188,8 +207,10 @@ private void accumulateChunk(String data) { private void finalizeSpan() { try { Long ttft = timeToFirstTokenNanos.get(); - InstrumentationSemConv.tagLLMSpanResponse( - span, providerName, accumulator.build(), ttft); + String responseBody = accumulator.build(); + InstrumentationSemConv.tagLLMSpanResponse(span, providerName, responseBody, ttft); + InstrumentationSemConv.addServerSideChildSpans( + tracer, span, providerName, responseBody, llmSpanStart); } catch (Exception e) { log.debug("Failed to finalize streaming span", e); } diff --git a/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java b/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java index 6f9fac29..1b43ab10 100644 --- a/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java +++ b/braintrust-sdk/instrumentation/openai_2_15_0/src/main/java/dev/braintrust/instrumentation/openai/v2_15_0/TracingHttpClient.java @@ -20,6 +20,7 @@ import io.opentelemetry.context.Context; import java.io.*; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -49,10 +50,11 @@ public TracingHttpClient(OpenTelemetry openTelemetry, HttpClient underlying) { * a long-lived client wrapped inside some unrelated span would otherwise parent every future * request to that stale span. */ - private Span startLlmSpan(@Nullable Context headerContext) { + private Span startLlmSpan(@Nullable Context headerContext, Instant startTime) { Context parent = headerContext != null ? headerContext : Context.current(); return tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME) .setParent(parent) + .setStartTimestamp(startTime) .startSpan(); } @@ -108,7 +110,8 @@ public void close() { public @NonNull HttpResponse execute( @NonNull HttpRequest httpRequest, @NonNull RequestOptions requestOptions) { var extracted = extractCallerContext(httpRequest); - var span = startLlmSpan(extracted.callerContext()); + var llmSpanStart = Instant.now(); + var span = startLlmSpan(extracted.callerContext(), llmSpanStart); try (var ignored = span.makeCurrent()) { // Buffer the request body so we can (a) read its bytes for the span attribute and // (b) supply a fresh, repeatable body to the underlying client — avoiding any @@ -130,7 +133,7 @@ public void close() { var response = underlying.execute(bufferedRequest, requestOptions); // Always tee the response body. onStreamClosed() detects whether the collected // bytes are SSE or plain JSON and tags the span accordingly. - return new TeeingStreamHttpResponse(response, span); + return new TeeingStreamHttpResponse(response, span, tracer, llmSpanStart); } catch (Exception e) { InstrumentationSemConv.tagLLMSpanResponse(span, e); span.end(); @@ -142,7 +145,8 @@ public void close() { public @NonNull CompletableFuture executeAsync( @NonNull HttpRequest httpRequest, @NonNull RequestOptions requestOptions) { var extracted = extractCallerContext(httpRequest); - var span = startLlmSpan(extracted.callerContext()); + var llmSpanStart = Instant.now(); + var span = startLlmSpan(extracted.callerContext(), llmSpanStart); try { var bufferedRequest = bufferRequestBody(extracted.request()); String inputJson = @@ -159,7 +163,10 @@ public void close() { return underlying .executeAsync(bufferedRequest, requestOptions) .thenApply( - response -> (HttpResponse) new TeeingStreamHttpResponse(response, span)) + response -> + (HttpResponse) + new TeeingStreamHttpResponse( + response, span, tracer, llmSpanStart)) .whenComplete( (response, t) -> { if (t != null) { @@ -239,21 +246,22 @@ private static String readBodyAsString(HttpRequestBody body) { * the bytes are an SSE stream (first non-empty line starts with {@code "data: "}) or a plain * JSON response, and parses accordingly. */ - private static void tagSpanFromBuffer(Span span, byte[] bytes, Long timeToFirstTokenNanos) { - if (bytes.length == 0) return; + private static String tagSpanFromBuffer(Span span, byte[] bytes, Long timeToFirstTokenNanos) { + if (bytes.length == 0) return null; try { String firstLine = firstNonEmptyLine(bytes); if (firstLine != null && (firstLine.startsWith("data:") || firstLine.startsWith("event:"))) { - tagSpanFromSseBytes(span, bytes, timeToFirstTokenNanos); + return tagSpanFromSseBytes(span, bytes, timeToFirstTokenNanos); } else { + String responseJson = new String(bytes, StandardCharsets.UTF_8); InstrumentationSemConv.tagLLMSpanResponse( - span, - InstrumentationSemConv.PROVIDER_NAME_OPENAI, - new String(bytes, StandardCharsets.UTF_8)); + span, InstrumentationSemConv.PROVIDER_NAME_OPENAI, responseJson); + return responseJson; } } catch (Exception e) { log.error("Could not tag span from response buffer", e); + return null; } } @@ -273,7 +281,7 @@ private static String firstNonEmptyLine(byte[] bytes) { * Parses SSE wire bytes, feeds each {@code data:} chunk through {@link * ChatCompletionAccumulator}, then tags the span with the reassembled output JSON. */ - private static void tagSpanFromSseBytes( + private static String tagSpanFromSseBytes( Span span, byte[] sseBytes, Long timeToFirstTokenNanos) { try { var reader = @@ -330,8 +338,10 @@ private static void tagSpanFromSseBytes( responseJson, timeToFirstTokenNanos); } + return responseJson; } catch (Exception e) { log.error("Could not parse SSE buffer to tag streaming span output", e); + return null; } } @@ -343,14 +353,19 @@ private static void tagSpanFromSseBytes( private static final class TeeingStreamHttpResponse implements HttpResponse { private final HttpResponse delegate; private final Span span; + private final Tracer tracer; + private final Instant llmSpanStart; private final long spanStartNanos = System.nanoTime(); private final AtomicLong timeToFirstTokenNanos = new AtomicLong(); private final ByteArrayOutputStream teeBuffer = new ByteArrayOutputStream(); private final InputStream teeStream; - TeeingStreamHttpResponse(HttpResponse delegate, Span span) { + TeeingStreamHttpResponse( + HttpResponse delegate, Span span, Tracer tracer, Instant llmSpanStart) { this.delegate = delegate; this.span = span; + this.tracer = tracer; + this.llmSpanStart = llmSpanStart; this.teeStream = new TeeInputStream( delegate.body(), teeBuffer, this::onFirstByte, this::onStreamClosed); @@ -369,7 +384,19 @@ private void onStreamClosed() { synchronized (teeBuffer) { bytes = teeBuffer.toByteArray(); } - tagSpanFromBuffer(span, bytes, timeToFirstTokenNanos.get()); + String responseJson = tagSpanFromBuffer(span, bytes, timeToFirstTokenNanos.get()); + if (responseJson != null) { + // Emit child spans for server-side tool calls (web search, etc.) nested under + // the LLM span, while it is still live. No-op for Chat Completions responses + // (no `output` array). Anchored at the LLM span start with zero duration — + // providers don't report per-tool timing, so we don't fabricate a duration. + InstrumentationSemConv.addServerSideChildSpans( + tracer, + span, + InstrumentationSemConv.PROVIDER_NAME_OPENAI, + responseJson, + llmSpanStart); + } } finally { span.end(); } diff --git a/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java index b34e5fc7..218a4175 100644 --- a/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java +++ b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAITest.java @@ -545,6 +545,8 @@ void testResponsesStreamingWithTools() { .toList(); assertFalse(functionCalls.isEmpty(), "model should call a function tool"); + // function_call is a client-side tool call — it stays in the LLM span output and does NOT + // produce a child span (only server-side tool calls do). So we expect exactly one span. var spans = testHarness.awaitExportedSpans(); assertEquals(1, spans.size()); var span = spans.get(0); diff --git a/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAIWebSearchTest.java b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAIWebSearchTest.java new file mode 100644 index 00000000..ae5111cf --- /dev/null +++ b/braintrust-sdk/instrumentation/openai_2_15_0/src/test/java/dev/braintrust/instrumentation/openai/v2_15_0/BraintrustOpenAIWebSearchTest.java @@ -0,0 +1,143 @@ +package dev.braintrust.instrumentation.openai.v2_15_0; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.client.OpenAIClient; +import com.openai.client.okhttp.OpenAIOkHttpClient; +import com.openai.core.http.StreamResponse; +import com.openai.helpers.ResponseAccumulator; +import com.openai.models.ChatModel; +import com.openai.models.responses.EasyInputMessage; +import com.openai.models.responses.Response; +import com.openai.models.responses.ResponseCreateParams; +import com.openai.models.responses.ResponseInputItem; +import com.openai.models.responses.ResponseStreamEvent; +import com.openai.models.responses.WebSearchTool; +import dev.braintrust.TestHarness; +import dev.braintrust.instrumentation.Instrumenter; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.trace.data.SpanData; +import java.util.List; +import lombok.SneakyThrows; +import net.bytebuddy.agent.ByteBuddyAgent; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Verifies that built-in web-search tool calls in the OpenAI Responses API are captured as child + * {@code type:"tool"} spans parented to the LLM span, giving web search its own cost/latency + * visibility on the trace timeline. + */ +public class BraintrustOpenAIWebSearchTest { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + private static final AttributeKey SPAN_ATTRIBUTES = + AttributeKey.stringKey("braintrust.span_attributes"); + private static final AttributeKey METADATA = + AttributeKey.stringKey("braintrust.metadata"); + + @BeforeAll + public static void beforeAll() { + var instrumentation = ByteBuddyAgent.install(); + Instrumenter.install(instrumentation, BraintrustOpenAIWebSearchTest.class.getClassLoader()); + } + + private TestHarness testHarness; + + @BeforeEach + void beforeEach() { + testHarness = TestHarness.setup(); + } + + private static ResponseCreateParams webSearchRequest() { + return ResponseCreateParams.builder() + .model(ChatModel.GPT_4O) + .inputOfResponse( + List.of( + ResponseInputItem.ofEasyInputMessage( + EasyInputMessage.builder() + .role(EasyInputMessage.Role.USER) + .content( + "What is one recent headline about" + + " artificial intelligence? Use web" + + " search.") + .build()))) + .addTool( + WebSearchTool.builder().type(WebSearchTool.Type.WEB_SEARCH_PREVIEW).build()) + .build(); + } + + @Test + @SneakyThrows + void testResponsesWebSearch() { + OpenAIClient client = + OpenAIOkHttpClient.builder() + .baseUrl(testHarness.openAiBaseUrl()) + .apiKey(testHarness.openAiApiKey()) + .build(); + + Response response = client.responses().create(webSearchRequest()); + assertNotNull(response); + + var spans = testHarness.awaitExportedSpans(2); + assertWebSearchToolSpans(spans); + } + + @Test + @SneakyThrows + void testResponsesWebSearchStreaming() { + OpenAIClient client = + OpenAIOkHttpClient.builder() + .baseUrl(testHarness.openAiBaseUrl()) + .apiKey(testHarness.openAiApiKey()) + .build(); + + var accumulator = ResponseAccumulator.create(); + try (StreamResponse stream = + client.responses().createStreaming(webSearchRequest())) { + stream.stream().forEach(accumulator::accumulate); + } + assertFalse(accumulator.response().output().isEmpty(), "should generate a response"); + + var spans = testHarness.awaitExportedSpans(2); + assertWebSearchToolSpans(spans); + } + + @SneakyThrows + private static void assertWebSearchToolSpans(List spans) { + // Exactly one LLM span (the Responses request), plus one or more tool spans. + var llmSpans = spans.stream().filter(s -> isType(s, "llm")).toList(); + assertEquals(1, llmSpans.size(), "expected a single LLM span"); + var llm = llmSpans.get(0); + + var webSearchSpans = + spans.stream() + .filter(s -> isType(s, "tool")) + .filter(s -> "web_search_call".equals(s.getName())) + .toList(); + assertFalse( + webSearchSpans.isEmpty(), + "expected at least one web_search_call tool span, got spans: " + + spans.stream().map(SpanData::getName).toList()); + + for (var ws : webSearchSpans) { + assertEquals( + llm.getSpanId(), + ws.getParentSpanId(), + "web_search_call tool span must be a child of the LLM span"); + JsonNode metadata = JSON_MAPPER.readTree(ws.getAttributes().get(METADATA)); + assertEquals("web_search_call", metadata.path("tool_type").asText()); + } + } + + @SneakyThrows + private static boolean isType(SpanData span, String type) { + String attr = span.getAttributes().get(SPAN_ATTRIBUTES); + if (attr == null) { + return false; + } + return type.equals(JSON_MAPPER.readTree(attr).path("type").asText()); + } +} diff --git a/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java b/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java index 7e9a7833..f3fa4e3a 100644 --- a/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java +++ b/braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java @@ -12,6 +12,7 @@ import java.lang.reflect.Field; import java.net.URI; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; @@ -180,7 +181,11 @@ static class BraintrustRestInterceptor implements ClientHttpRequestInterceptor { public ClientHttpResponse intercept( HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { - Span span = tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME).startSpan(); + Instant llmSpanStart = Instant.now(); + Span span = + tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME) + .setStartTimestamp(llmSpanStart) + .startSpan(); try { String requestBody = new String(body, StandardCharsets.UTF_8); List pathSegments = extractPathSegments(request.getURI()); @@ -199,6 +204,8 @@ public ClientHttpResponse intercept( String responseBody = new String(responseBytes, StandardCharsets.UTF_8); InstrumentationSemConv.tagLLMSpanResponse(span, providerName, responseBody); + InstrumentationSemConv.addServerSideChildSpans( + tracer, span, providerName, responseBody, llmSpanStart); span.end(); return new BufferedClientHttpResponse(response, responseBytes); @@ -271,7 +278,11 @@ static class BraintrustWebClientFilter implements ExchangeFilterFunction { @Override public Mono filter(ClientRequest request, ExchangeFunction next) { - Span span = tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME).startSpan(); + Instant llmSpanStart = Instant.now(); + Span span = + tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME) + .setStartTimestamp(llmSpanStart) + .startSpan(); long startNanos = System.nanoTime(); List pathSegments = extractPathSegments(request.url()); @@ -307,6 +318,7 @@ public Mono filter(ClientRequest request, ExchangeFunction next) originalBody, span, startNanos, + llmSpanStart, streamCtx)) .build(); }) @@ -331,6 +343,7 @@ private Flux wrapStreamingBody( Publisher originalBody, Span span, long startNanos, + Instant llmSpanStart, StreamContext streamCtx) { final long[] ttftNanos = {-1}; StringBuilder assembled = new StringBuilder(); @@ -359,6 +372,12 @@ private Flux wrapStreamingBody( Long ttft = ttftNanos[0] >= 0 ? ttftNanos[0] : null; InstrumentationSemConv.tagLLMSpanResponse( span, streamCtx.providerName(), responseBody, ttft); + InstrumentationSemConv.addServerSideChildSpans( + tracer, + span, + streamCtx.providerName(), + responseBody, + llmSpanStart); } catch (Exception e) { log.debug("failed to tag streaming response", e); } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java index 1dc25cfe..ef646fa4 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java @@ -4,12 +4,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import dev.braintrust.json.BraintrustJsonMapper; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import java.time.Instant; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.SneakyThrows; @@ -103,6 +109,40 @@ public static void tagLLMSpanResponse(Span span, @Nonnull Throwable responseErro span.recordException(responseError); } + /** + * Emit child {@code type:"tool"} spans for built-in tool calls the vendor executed server + * side (web search, file search, code interpreter, image generation, remote MCP) that the + * provider reports inline with the LLM response. These are otherwise invisible on the trace — + * unlike client-side tool calls (a plain {@code function_call}/{@code computer_call}), which + * the caller executes and which get instrumented where they run — so they are surfaced as + * children of the LLM span. + * + *

    Each child is a zero-duration marker anchored at {@code spanStart} (the parent LLM span's + * start): providers don't report per-tool timing, so we deliberately do not fabricate a + * duration. Safe to call for any response — non-matching payloads simply yield no spans. + * + * @param tracer used to create the child spans; cannot be derived from {@code llmSpan} + * @param llmSpan the parent LLM span the children are nested under + * @param spanStart the parent LLM span's start instant + */ + public static void addServerSideChildSpans( + @Nonnull Tracer tracer, + @Nonnull Span llmSpan, + @Nonnull String providerName, + @Nonnull String responseBody, + @Nonnull Instant spanStart) { + Context parentContext = Context.current().with(llmSpan); + switch (providerName) { + case PROVIDER_NAME_OPENAI -> + addOpenAIServerSideChildSpans(tracer, parentContext, responseBody, spanStart); + case PROVIDER_NAME_ANTHROPIC -> + addAnthropicServerSideChildSpans( + tracer, parentContext, responseBody, spanStart); + default -> + addOpenAIServerSideChildSpans(tracer, parentContext, responseBody, spanStart); + } + } + // ------------------------------------------------------------------------- // OpenAI provider implementation // ------------------------------------------------------------------------- @@ -207,6 +247,158 @@ private static void tagOpenAIResponse( } } + private static final String TYPE_TOOL_JSON = "{\"type\":\"tool\"}"; + + /** + * OpenAI Responses {@code output} item types the vendor executes server side, mapped + * to the item fields that make up the tool span's input. Client-side calls ({@code + * function_call}, {@code computer_call}) are intentionally excluded — they run in the caller + * and are instrumented there. + */ + private static final Map> OPENAI_SERVER_SIDE_ITEM_INPUT_KEYS = + Map.of( + "web_search_call", List.of("action"), + "file_search_call", List.of("queries"), + "code_interpreter_call", List.of("code", "container_id"), + "image_generation_call", List.of(), + "mcp_call", List.of("arguments")); + + private static void addOpenAIServerSideChildSpans( + Tracer tracer, Context parentContext, String responseBody, Instant spanStart) { + try { + JsonNode root = BraintrustJsonMapper.get().readTree(responseBody); + JsonNode output = root.get("output"); + if (output == null || !output.isArray()) { + return; + } + for (JsonNode item : output) { + if (!item.isObject()) { + continue; + } + String type = item.path("type").asText(null); + if (type == null || !OPENAI_SERVER_SIDE_ITEM_INPUT_KEYS.containsKey(type)) { + continue; + } + emitOpenAIServerSideToolSpan(tracer, parentContext, item, type, spanStart); + } + } catch (Exception e) { + log.debug("Could not emit OpenAI server-side child spans", e); + } + } + + private static void emitOpenAIServerSideToolSpan( + Tracer tracer, Context parentContext, JsonNode item, String type, Instant spanStart) { + Span span = + tracer.spanBuilder(openAIToolSpanName(item, type)) + .setParent(parentContext) + .setStartTimestamp(spanStart) + .startSpan(); + try { + span.setAttribute("braintrust.span_attributes", TYPE_TOOL_JSON); + + JsonNode input = openAIToolSpanInput(item, type); + if (input != null && !input.isNull()) { + span.setAttribute("braintrust.input_json", toJson(input)); + } + String metadata = openAIToolSpanMetadata(item, type); + if (metadata != null) { + span.setAttribute("braintrust.metadata", metadata); + } + + // When the tool call errored, record the error and skip output. + JsonNode error = openAIToolSpanError(item); + if (error != null) { + span.setStatus( + StatusCode.ERROR, error.isValueNode() ? error.asText() : error.toString()); + return; + } + JsonNode output = openAIToolSpanOutput(item, type); + if (output != null) { + span.setAttribute("braintrust.output_json", toJson(output)); + } + } catch (Exception e) { + log.debug("Could not tag OpenAI server-side tool span", e); + } finally { + // Zero-duration marker: end at the same instant it started. + span.end(spanStart); + } + } + + private static String openAIToolSpanName(JsonNode item, String type) { + String serverLabel = nonEmptyText(item, "server_label"); + String name = nonEmptyText(item, "name"); + if (serverLabel != null && name != null) { + return serverLabel + "." + name; + } + if (name != null) { + return name; + } + return type; + } + + private static JsonNode openAIToolSpanInput(JsonNode item, String type) { + List inputKeys = OPENAI_SERVER_SIDE_ITEM_INPUT_KEYS.get(type); + if (inputKeys.isEmpty()) { + return null; + } + ObjectNode inputData = BraintrustJsonMapper.get().createObjectNode(); + for (String key : inputKeys) { + JsonNode value = item.get(key); + if (value != null && !value.isNull()) { + inputData.set(key, maybeParseJsonString(value)); + } + } + if (inputData.isEmpty()) { + return null; + } + // MCP calls carry a single `arguments` blob — unwrap it to the bare value. + if (inputKeys.size() == 1 && "arguments".equals(inputKeys.get(0))) { + return inputData.get("arguments"); + } + return inputData; + } + + private static JsonNode openAIToolSpanOutput(JsonNode item, String type) { + Set excluded = + new HashSet<>(Set.of("id", "type", "name", "call_id", "server_label", "error")); + excluded.addAll(OPENAI_SERVER_SIDE_ITEM_INPUT_KEYS.get(type)); + + ObjectNode output = BraintrustJsonMapper.get().createObjectNode(); + var fields = item.fields(); + while (fields.hasNext()) { + var entry = fields.next(); + String key = entry.getKey(); + JsonNode value = entry.getValue(); + if (excluded.contains(key) || value == null || value.isNull()) { + continue; + } + if ("output".equals(key) || "error".equals(key)) { + output.set(key, maybeParseJsonString(value)); + } else { + output.set(key, value); + } + } + return output.isEmpty() ? null : output; + } + + private static JsonNode openAIToolSpanError(JsonNode item) { + JsonNode error = item.get("error"); + if (error == null || error.isNull()) { + return null; + } + return maybeParseJsonString(error); + } + + private static String openAIToolSpanMetadata(JsonNode item, String type) { + ObjectNode md = BraintrustJsonMapper.get().createObjectNode(); + md.put("tool_type", type); + putIfPresent(md, "tool_id", item.get("id")); + putIfPresent(md, "call_id", item.get("call_id")); + putIfPresent(md, "status", item.get("status")); + putIfPresent(md, "server_label", item.get("server_label")); + return md.isEmpty() ? null : toJson(md); + } + // ------------------------------------------------------------------------- // Anthropic provider implementation // ------------------------------------------------------------------------- @@ -300,6 +492,19 @@ private static void tagAnthropicResponse( metrics.put("prompt_cache_creation_tokens", cacheCreationTokens); } } + + // Server-side tool usage counts (e.g. web_search_requests, web_fetch_requests). + // Each numeric field becomes a server_tool_use_ metric the backend prices — + // this is how web search cost is attributed. + if (usage.has("server_tool_use") && usage.get("server_tool_use").isObject()) { + var fields = usage.get("server_tool_use").fields(); + while (fields.hasNext()) { + var entry = fields.next(); + if (entry.getValue().isNumber()) { + metrics.put("server_tool_use_" + entry.getKey(), entry.getValue()); + } + } + } } if (!metrics.isEmpty()) { @@ -339,6 +544,267 @@ private static boolean addPerTtlCacheMetrics(Map metrics, JsonNo return emitted; } + private static final String ANTHROPIC_SERVER_TOOL_USE_TYPE = "server_tool_use"; + private static final String ANTHROPIC_TOOL_RESULT_SUFFIX = "_tool_result"; + + /** + * Emit child tool spans for Anthropic server-side tool use. In the Message {@code content} + * array these appear as a {@code server_tool_use} block (the call) and a matching {@code + * *_tool_result} block (e.g. {@code web_search_tool_result}), linked by {@code id} /{@code + * tool_use_id}. Calls and results are paired (buffering results that arrive before their call); + * unmatched calls and results each still get a span. Mirrors the Python SDK's {@code + * _log_server_tool_spans}. + */ + private static void addAnthropicServerSideChildSpans( + Tracer tracer, Context parentContext, String responseBody, Instant spanStart) { + try { + JsonNode content = BraintrustJsonMapper.get().readTree(responseBody).get("content"); + if (content == null || !content.isArray()) { + return; + } + Map callsById = new java.util.LinkedHashMap<>(); + Map> pendingResultsById = new java.util.LinkedHashMap<>(); + Set matchedCallIds = new HashSet<>(); + List pairs = new java.util.ArrayList<>(); // {call, result}, either nullable + + for (JsonNode item : content) { + if (!item.isObject()) { + continue; + } + String itemType = item.path("type").asText(null); + if (ANTHROPIC_SERVER_TOOL_USE_TYPE.equals(itemType)) { + JsonNode id = item.get("id"); + if (id != null && id.isTextual()) { + callsById.put(id.asText(), item); + List pending = pendingResultsById.remove(id.asText()); + if (pending != null) { + for (JsonNode result : pending) { + pairs.add(new JsonNode[] {item, result}); + matchedCallIds.add(id.asText()); + } + } + } else { + pairs.add(new JsonNode[] {item, null}); + } + } else if (isAnthropicToolResultType(itemType)) { + JsonNode toolUseId = item.get("tool_use_id"); + if (toolUseId != null && toolUseId.isTextual()) { + if (callsById.containsKey(toolUseId.asText())) { + pairs.add(new JsonNode[] {callsById.get(toolUseId.asText()), item}); + matchedCallIds.add(toolUseId.asText()); + } else { + pendingResultsById + .computeIfAbsent( + toolUseId.asText(), k -> new java.util.ArrayList<>()) + .add(item); + } + } else { + pairs.add(new JsonNode[] {null, item}); + } + } + } + + for (JsonNode[] pair : pairs) { + emitAnthropicServerToolSpan(tracer, parentContext, pair[0], pair[1], spanStart); + } + for (Map.Entry entry : callsById.entrySet()) { + if (!matchedCallIds.contains(entry.getKey())) { + emitAnthropicServerToolSpan( + tracer, parentContext, entry.getValue(), null, spanStart); + } + } + for (List pending : pendingResultsById.values()) { + for (JsonNode result : pending) { + emitAnthropicServerToolSpan(tracer, parentContext, null, result, spanStart); + } + } + } catch (Exception e) { + log.debug("Could not emit Anthropic server-side child spans", e); + } + } + + private static boolean isAnthropicToolResultType(@Nullable String type) { + return type != null + && type.endsWith(ANTHROPIC_TOOL_RESULT_SUFFIX) + && !type.equals("tool_result"); + } + + private static void emitAnthropicServerToolSpan( + Tracer tracer, + Context parentContext, + @Nullable JsonNode call, + @Nullable JsonNode result, + Instant spanStart) { + Span span = + tracer.spanBuilder(anthropicToolSpanName(call, result)) + .setParent(parentContext) + .setStartTimestamp(spanStart) + .startSpan(); + try { + span.setAttribute("braintrust.span_attributes", TYPE_TOOL_JSON); + + JsonNode input = anthropicToolSpanInput(call); + if (input != null && !input.isNull()) { + span.setAttribute("braintrust.input_json", toJson(input)); + } + String metadata = anthropicToolSpanMetadata(call, result); + if (metadata != null) { + span.setAttribute("braintrust.metadata", metadata); + } + + JsonNode output = anthropicToolSpanOutput(result); + if (output == null || output.isNull()) { + return; // no result content — input + metadata only (matches Python) + } + String error = anthropicToolSpanError(result); + if (error != null) { + span.setStatus(StatusCode.ERROR, error); + } + span.setAttribute("braintrust.output_json", toJson(output)); + } catch (Exception e) { + log.debug("Could not tag Anthropic server-side tool span", e); + } finally { + // Zero-duration marker: end at the same instant it started. + span.end(spanStart); + } + } + + private static String anthropicToolSpanName( + @Nullable JsonNode call, @Nullable JsonNode result) { + if (call != null) { + JsonNode name = call.get("name"); + if (name != null && name.isTextual()) { + return name.asText(); + } + } + if (result != null) { + JsonNode type = result.get("type"); + if (type != null + && type.isTextual() + && type.asText().endsWith(ANTHROPIC_TOOL_RESULT_SUFFIX)) { + String t = type.asText(); + return t.substring(0, t.length() - ANTHROPIC_TOOL_RESULT_SUFFIX.length()); + } + } + return "server_tool"; + } + + private static final Set ANTHROPIC_CALL_INPUT_EXCLUDED = + Set.of("id", "type", "name", "caller"); + + private static JsonNode anthropicToolSpanInput(@Nullable JsonNode call) { + if (call == null) { + return null; + } + JsonNode input = call.get("input"); + if (input != null && !input.isNull()) { + return input; + } + ObjectNode obj = BraintrustJsonMapper.get().createObjectNode(); + var fields = call.fields(); + while (fields.hasNext()) { + var entry = fields.next(); + if (!ANTHROPIC_CALL_INPUT_EXCLUDED.contains(entry.getKey())) { + obj.set(entry.getKey(), entry.getValue()); + } + } + return obj.isEmpty() ? null : obj; + } + + private static final Set ANTHROPIC_RESULT_OUTPUT_EXCLUDED = + Set.of("tool_use_id", "type", "caller"); + + private static JsonNode anthropicToolSpanOutput(@Nullable JsonNode result) { + if (result == null) { + return null; + } + if (result.has("content")) { + return redactServerToolOutput(result.get("content")); + } + ObjectNode obj = BraintrustJsonMapper.get().createObjectNode(); + var fields = result.fields(); + while (fields.hasNext()) { + var entry = fields.next(); + if (!ANTHROPIC_RESULT_OUTPUT_EXCLUDED.contains(entry.getKey())) { + obj.set(entry.getKey(), redactServerToolOutput(entry.getValue())); + } + } + return obj.isEmpty() ? null : obj; + } + + /** Recursively replace {@code encrypted_content} values (opaque, large) with a placeholder. */ + private static JsonNode redactServerToolOutput(JsonNode value) { + if (value == null) { + return null; + } + if (value.isArray()) { + ArrayNode arr = BraintrustJsonMapper.get().createArrayNode(); + for (JsonNode item : value) { + arr.add(redactServerToolOutput(item)); + } + return arr; + } + if (value.isObject()) { + ObjectNode obj = BraintrustJsonMapper.get().createObjectNode(); + var fields = value.fields(); + while (fields.hasNext()) { + var entry = fields.next(); + if ("encrypted_content".equals(entry.getKey())) { + obj.put(entry.getKey(), ""); + } else { + obj.set(entry.getKey(), redactServerToolOutput(entry.getValue())); + } + } + return obj; + } + return value; + } + + private static String anthropicToolSpanError(@Nullable JsonNode result) { + if (result == null) { + return null; + } + JsonNode content = result.get("content"); + if (content == null || !content.isObject()) { + return null; + } + JsonNode type = content.get("type"); + if (type == null || !type.isTextual() || !type.asText().endsWith("_error")) { + return null; + } + JsonNode message = content.get("error_message"); + if (message != null && message.isTextual() && !message.asText().isEmpty()) { + return message.asText(); + } + JsonNode code = content.get("error_code"); + if (code != null && code.isTextual() && !code.asText().isEmpty()) { + return code.asText(); + } + return type.asText(); + } + + private static String anthropicToolSpanMetadata( + @Nullable JsonNode call, @Nullable JsonNode result) { + ObjectNode md = BraintrustJsonMapper.get().createObjectNode(); + JsonNode toolUseId = call != null ? call.get("id") : null; + if (toolUseId == null || toolUseId.isNull()) { + toolUseId = result != null ? result.get("tool_use_id") : null; + } + putIfPresent(md, "tool_use_id", toolUseId); + if (call != null) { + putIfPresent(md, "tool_call_type", call.get("type")); + } + if (result != null) { + putIfPresent(md, "tool_result_type", result.get("type")); + } + JsonNode caller = call != null ? call.get("caller") : null; + if (caller == null || caller.isNull()) { + caller = result != null ? result.get("caller") : null; + } + putIfPresent(md, "caller", caller); + return md.isEmpty() ? null : toJson(md); + } + // ------------------------------------------------------------------------- // AWS Bedrock provider implementation // ------------------------------------------------------------------------- @@ -439,6 +905,42 @@ private static void tagBedrockResponse( // Shared helpers // ------------------------------------------------------------------------- + /** + * If {@code value} is a string that looks like JSON (starts with {@code [} or {@code {}), parse + * and return it as a tree; otherwise return it unchanged. + */ + private static JsonNode maybeParseJsonString(JsonNode value) { + if (value == null || !value.isTextual()) { + return value; + } + String stripped = value.asText().strip(); + if (stripped.isEmpty() || (stripped.charAt(0) != '[' && stripped.charAt(0) != '{')) { + return value; + } + try { + return BraintrustJsonMapper.get().readTree(stripped); + } catch (Exception e) { + return value; + } + } + + private static void putIfPresent(ObjectNode target, String key, JsonNode value) { + if (value != null && !value.isNull()) { + target.set(key, value); + } + } + + private static String nonEmptyText(JsonNode item, String field) { + JsonNode node = item.get(field); + if (node != null && node.isValueNode()) { + String text = node.asText(); + if (!text.isEmpty()) { + return text; + } + } + return null; + } + /** * Simplifies an Anthropic message node by converting single-text content block arrays (e.g. * {@code [{"type":"text","text":"hello"}]}) to plain strings. This normalizes the format used diff --git a/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvChildSpansTest.java b/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvChildSpansTest.java new file mode 100644 index 00000000..d222372d --- /dev/null +++ b/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvChildSpansTest.java @@ -0,0 +1,355 @@ +package dev.braintrust.instrumentation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import dev.braintrust.json.BraintrustJsonMapper; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Covers {@link InstrumentationSemConv#addServerSideChildSpans}: only vendor-executed (server-side) + * tool calls in an OpenAI Responses body become child spans nested under the LLM span. Client-side + * calls ({@code function_call}, {@code computer_call}) are excluded. + */ +class InstrumentationSemConvChildSpansTest { + + private static final AttributeKey SPAN_ATTRIBUTES = + AttributeKey.stringKey("braintrust.span_attributes"); + private static final AttributeKey INPUT_JSON = + AttributeKey.stringKey("braintrust.input_json"); + private static final AttributeKey OUTPUT_JSON = + AttributeKey.stringKey("braintrust.output_json"); + private static final AttributeKey METADATA = + AttributeKey.stringKey("braintrust.metadata"); + + /** Fixed LLM-span start the child spans are anchored to, so we can assert on it. */ + private static final java.time.Instant LLM_START = + java.time.Instant.ofEpochSecond(1_700_000_000L); + + private InMemorySpanExporter exporter; + private SdkTracerProvider tracerProvider; + private Tracer tracer; + + @BeforeEach + void setUp() { + exporter = InMemorySpanExporter.create(); + tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build(); + tracer = tracerProvider.get("test"); + } + + @AfterEach + void tearDown() { + tracerProvider.close(); + } + + /** Runs the emitter with an LLM parent span and returns only the emitted child spans. */ + private List emitOpenAI(String responseBody) { + Span parent = tracer.spanBuilder("llm").setStartTimestamp(LLM_START).startSpan(); + try (var ignored = parent.makeCurrent()) { + InstrumentationSemConv.addServerSideChildSpans( + tracer, + parent, + InstrumentationSemConv.PROVIDER_NAME_OPENAI, + responseBody, + LLM_START); + } finally { + parent.end(); + } + return exporter.getFinishedSpanItems().stream() + .filter(s -> !s.getName().equals("llm")) + .collect(Collectors.toList()); + } + + private static SpanData byName(List spans, String name) { + return spans.stream().filter(s -> s.getName().equals(name)).findFirst().orElseThrow(); + } + + private static JsonNode json(String s) { + return BraintrustJsonMapper.fromJson(s, JsonNode.class); + } + + @Test + void emitsWebSearchCallToolSpanParentedToLlm() { + String body = + """ + { + "id": "resp_1", + "output": [ + { + "id": "ws_1", + "type": "web_search_call", + "status": "completed", + "action": {"type": "search", "query": "braintrust observability"} + }, + { + "id": "msg_1", + "type": "message", + "content": [{"type": "output_text", "text": "here is the answer"}] + } + ], + "usage": {"input_tokens": 10, "output_tokens": 20} + } + """; + + List tools = emitOpenAI(body); + // Only the web_search_call becomes a tool span; the message item is ignored. + assertEquals(1, tools.size()); + + SpanData ws = byName(tools, "web_search_call"); + // Parented to the LLM span. + SpanData llm = byName(exporter.getFinishedSpanItems(), "llm"); + assertEquals(llm.getSpanId(), ws.getParentSpanId()); + + assertEquals("{\"type\":\"tool\"}", ws.getAttributes().get(SPAN_ATTRIBUTES)); + + // action is the sole input key -> object with the action content. + JsonNode input = json(ws.getAttributes().get(INPUT_JSON)); + assertEquals("search", input.path("action").path("type").asText()); + assertEquals("braintrust observability", input.path("action").path("query").asText()); + + // metadata carries tool_type / tool_id / status. + JsonNode metadata = json(ws.getAttributes().get(METADATA)); + assertEquals("web_search_call", metadata.path("tool_type").asText()); + assertEquals("ws_1", metadata.path("tool_id").asText()); + assertEquals("completed", metadata.path("status").asText()); + + // action is an input key and id/type/error are excluded; the remaining `status` field + // flows into output (matching the Python SDK). + JsonNode output = json(ws.getAttributes().get(OUTPUT_JSON)); + assertEquals("completed", output.path("status").asText()); + + // Zero-duration marker anchored at the LLM span start — providers don't report per-tool + // timing, so we don't fabricate a duration. + long startNanos = LLM_START.getEpochSecond() * 1_000_000_000L + LLM_START.getNano(); + assertEquals(startNanos, ws.getStartEpochNanos()); + assertEquals(ws.getStartEpochNanos(), ws.getEndEpochNanos()); + } + + @Test + void clientSideCallsAreNotSpanned() { + // function_call and computer_call are executed by the caller, not the vendor — no spans. + String body = + """ + { + "output": [ + { + "id": "fc_1", + "type": "function_call", + "call_id": "call_1", + "name": "get_weather", + "arguments": "{\\"city\\": \\"sf\\"}" + }, + { + "id": "cc_1", + "type": "computer_call", + "action": {"type": "screenshot"} + } + ] + } + """; + assertTrue(emitOpenAI(body).isEmpty()); + } + + @Test + void errorItemSetsErrorStatusAndSkipsOutput() { + String body = + """ + { + "output": [ + { + "id": "ws_2", + "type": "web_search_call", + "status": "failed", + "action": {"query": "x"}, + "error": "rate_limited" + } + ] + } + """; + + List tools = emitOpenAI(body); + assertEquals(1, tools.size()); + SpanData ws = tools.get(0); + assertEquals(StatusCode.ERROR, ws.getStatus().getStatusCode()); + assertNull(ws.getAttributes().get(OUTPUT_JSON)); + } + + @Test + void mcpCallUsesServerLabelDottedNameAndUnwrapsArguments() { + String body = + """ + { + "output": [ + { + "id": "mcp_1", + "type": "mcp_call", + "server_label": "deepwiki", + "name": "ask_question", + "arguments": "{\\"q\\": \\"hi\\"}", + "output": "an answer" + } + ] + } + """; + + List tools = emitOpenAI(body); + assertEquals(1, tools.size()); + SpanData mcp = byName(tools, "deepwiki.ask_question"); + // arguments (sole input key) unwrapped to the bare, parsed value. + JsonNode input = json(mcp.getAttributes().get(INPUT_JSON)); + assertEquals("hi", input.path("q").asText()); + // output present (mcp_call is server-side) and JSON-parsed where applicable. + assertEquals( + "an answer", json(mcp.getAttributes().get(OUTPUT_JSON)).path("output").asText()); + } + + @Test + void nonToolResponseEmitsNothing() { + // A Chat Completions body (no `output` array) yields no child spans. + String body = + """ + {"choices": [{"message": {"content": "hi"}}], "usage": {"total_tokens": 5}} + """; + assertTrue(emitOpenAI(body).isEmpty()); + } + + @Test + void malformedBodyIsIgnored() { + assertTrue(emitOpenAI("not json").isEmpty()); + } + + // ------------------------------------------------------------------------- + // Anthropic server-side tool spans + // ------------------------------------------------------------------------- + + /** Runs the Anthropic emitter with an LLM parent span and returns only the child spans. */ + private List emitAnthropic(String responseBody) { + Span parent = tracer.spanBuilder("llm").setStartTimestamp(LLM_START).startSpan(); + try (var ignored = parent.makeCurrent()) { + InstrumentationSemConv.addServerSideChildSpans( + tracer, + parent, + InstrumentationSemConv.PROVIDER_NAME_ANTHROPIC, + responseBody, + LLM_START); + } finally { + parent.end(); + } + return exporter.getFinishedSpanItems().stream() + .filter(s -> !s.getName().equals("llm")) + .collect(Collectors.toList()); + } + + @Test + void pairsServerToolUseWithResultAndRedactsEncryptedContent() { + String body = + """ + { + "content": [ + {"type": "server_tool_use", "id": "srv_1", "name": "web_search", + "input": {"query": "braintrust"}}, + {"type": "web_search_tool_result", "tool_use_id": "srv_1", + "content": [{"type": "web_search_result", "title": "BT", + "encrypted_content": "SECRET-BLOB"}]}, + {"type": "text", "text": "here is the answer"} + ] + } + """; + + List tools = emitAnthropic(body); + assertEquals(1, tools.size()); + SpanData ws = byName(tools, "web_search"); + + // Parented to the LLM span. + SpanData llm = byName(exporter.getFinishedSpanItems(), "llm"); + assertEquals(llm.getSpanId(), ws.getParentSpanId()); + assertEquals("{\"type\":\"tool\"}", ws.getAttributes().get(SPAN_ATTRIBUTES)); + + // input = the call's `input`. + assertEquals("braintrust", json(ws.getAttributes().get(INPUT_JSON)).path("query").asText()); + + // metadata carries the pairing identity. + JsonNode md = json(ws.getAttributes().get(METADATA)); + assertEquals("srv_1", md.path("tool_use_id").asText()); + assertEquals("server_tool_use", md.path("tool_call_type").asText()); + assertEquals("web_search_tool_result", md.path("tool_result_type").asText()); + + // output = the result content, with encrypted_content redacted. + JsonNode output = json(ws.getAttributes().get(OUTPUT_JSON)); + assertEquals("BT", output.get(0).path("title").asText()); + assertEquals("", output.get(0).path("encrypted_content").asText()); + } + + @Test + void serverToolResultErrorSetsErrorStatus() { + String body = + """ + { + "content": [ + {"type": "server_tool_use", "id": "srv_2", "name": "web_search", + "input": {"query": "x"}}, + {"type": "web_search_tool_result", "tool_use_id": "srv_2", + "content": {"type": "web_search_tool_result_error", + "error_code": "max_uses_exceeded"}} + ] + } + """; + + List tools = emitAnthropic(body); + assertEquals(1, tools.size()); + assertEquals(StatusCode.ERROR, tools.get(0).getStatus().getStatusCode()); + } + + @Test + void unmatchedServerToolUseStillEmitsSpanWithoutOutput() { + // A call with no matching *_tool_result — input + metadata, no output. + String body = + """ + { + "content": [ + {"type": "server_tool_use", "id": "srv_3", "name": "web_search", + "input": {"query": "y"}} + ] + } + """; + + List tools = emitAnthropic(body); + assertEquals(1, tools.size()); + SpanData ws = tools.get(0); + assertEquals("web_search", ws.getName()); + assertNull(ws.getAttributes().get(OUTPUT_JSON)); + assertEquals("y", json(ws.getAttributes().get(INPUT_JSON)).path("query").asText()); + } + + @Test + void clientToolUseAndPlainTextEmitNothing() { + // Client-side tool_use blocks and text are not server-side tools. + String body = + """ + { + "content": [ + {"type": "tool_use", "id": "t1", "name": "get_weather", "input": {"city": "sf"}}, + {"type": "text", "text": "hi"} + ] + } + """; + assertTrue(emitAnthropic(body).isEmpty()); + } +} diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java index e5eebeb1..9768db00 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java @@ -73,10 +73,17 @@ public final class SpecClientRegistry { "springai2-anthropic", "anthropic", Set.of("/v1/messages"), - // Spec-level cache_control block placement isn't expressible - // via ChatModel messages (Spring AI 2.0 models caching through - // AnthropicCacheOptions instead). - Set.of("prompt_caching_5m", "prompt_caching_1h"), + // prompt_caching: cache_control block placement isn't + // expressible via ChatModel messages (Spring AI 2.0 models + // caching through AnthropicCacheOptions instead). + // web_search: the Spring AI framework (not our SDK) silently + // drops the native web_search_20250305 server tool when + // serializing the request (verified: it sends tools=null), so + // no + // search ever runs. Our instrumentation is correct — there is + // simply no server tool use to surface. Only the raw anthropic + // client exercises this spec. + Set.of("prompt_caching_5m", "prompt_caching_1h", "web_search"), new SpecClient.Isolation( "btx.springai2.classpath", "dev.braintrust.sdkspecimpl.springai2.SpringAi2AnthropicSpecClient"))) diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/SpringAi1AnthropicSpecClient.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/SpringAi1AnthropicSpecClient.java index 3ca6cb0d..87a4c03d 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/SpringAi1AnthropicSpecClient.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/SpringAi1AnthropicSpecClient.java @@ -25,9 +25,20 @@ public String provider() { return "anthropic"; } + /** + * Specs this client cannot express, keyed by spec {@code name}. The limitation is in the Spring + * AI framework, not our instrumentation: Spring AI 1.x's Anthropic response model ({@code + * AnthropicApi.ContentBlock.Type}) has no {@code web_search_tool_result} value, so the + * framework throws {@code HttpMessageNotReadableException} deserializing a web-search response + * — after our HTTP-layer instrumentation has already captured the spans correctly. (Spring AI + * 2.x fails differently: it silently drops the native tool from the request entirely.) Web + * search is therefore only exercised through the raw {@code anthropic} client. + */ + private static final java.util.Set UNSUPPORTED_SPECS = java.util.Set.of("web_search"); + @Override public boolean supports(LlmSpanSpec spec) { - return "/v1/messages".equals(spec.endpoint()); + return "/v1/messages".equals(spec.endpoint()) && !UNSUPPORTED_SPECS.contains(spec.name()); } @Override From e9d997c7ebaa372767f5e65d23f64221abf9e909 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Tue, 18 Aug 2026 11:15:38 -0600 Subject: [PATCH 4/4] wip --- .../langchain_1_14_0/build.gradle | 69 +++ .../v1_14_0/BraintrustLangchain.java | 242 +++++++++ .../v1_14_0/OtelContextPassingExecutor.java | 29 + .../langchain/v1_14_0/TracingProxy.java | 48 ++ .../v1_14_0/TracingToolExecutor.java | 78 +++ .../langchain/v1_14_0/WrappedHttpClient.java | 219 ++++++++ .../v1_14_0/WrappedHttpClientBuilder.java | 48 ++ .../auto/LangchainInstrumentationModule.java | 228 ++++++++ .../v1_14_0/BraintrustLangchainTest.java | 498 ++++++++++++++++++ .../v1_14_0/TracingToolExecutorTest.java | 18 + .../langchain_1_8_0/build.gradle | 15 +- .../auto/LangchainInstrumentationModule.java | 15 + btx/build.gradle | 11 +- .../braintrust/sdkspecimpl/SpanValidator.java | 36 +- .../sdkspecimpl/SpecClientRegistry.java | 2 + .../LangChainOpenAiResponsesSpecClient.java | 120 +++++ .../clients/LangChainOpenAiSpecClient.java | 7 +- settings.gradle | 1 + .../__files/responses-863a18378a48.json | 113 ++++ .../__files/responses-892784bdb435.json | 92 ++++ .../__files/responses-bdae47d54959.json | 113 ++++ .../__files/responses-cfec8f65e9bb.json | 100 ++++ .../__files/responses-f0262cdce49b.json | 109 ++++ .../mappings/responses-863a18378a48.json | 48 ++ .../mappings/responses-892784bdb435.json | 48 ++ .../mappings/responses-bdae47d54959.json | 48 ++ .../mappings/responses-cfec8f65e9bb.json | 48 ++ .../mappings/responses-f0262cdce49b.json | 48 ++ 28 files changed, 2440 insertions(+), 11 deletions(-) create mode 100644 braintrust-sdk/instrumentation/langchain_1_14_0/build.gradle create mode 100644 braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/BraintrustLangchain.java create mode 100644 braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/OtelContextPassingExecutor.java create mode 100644 braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingProxy.java create mode 100644 braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingToolExecutor.java create mode 100644 braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/WrappedHttpClient.java create mode 100644 braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/WrappedHttpClientBuilder.java create mode 100644 braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/auto/LangchainInstrumentationModule.java create mode 100644 braintrust-sdk/instrumentation/langchain_1_14_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_14_0/BraintrustLangchainTest.java create mode 100644 braintrust-sdk/instrumentation/langchain_1_14_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingToolExecutorTest.java create mode 100644 btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChainOpenAiResponsesSpecClient.java create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-863a18378a48.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-892784bdb435.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-bdae47d54959.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-cfec8f65e9bb.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-f0262cdce49b.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-863a18378a48.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-892784bdb435.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-bdae47d54959.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-cfec8f65e9bb.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-f0262cdce49b.json diff --git a/braintrust-sdk/instrumentation/langchain_1_14_0/build.gradle b/braintrust-sdk/instrumentation/langchain_1_14_0/build.gradle new file mode 100644 index 00000000..7b90ba08 --- /dev/null +++ b/braintrust-sdk/instrumentation/langchain_1_14_0/build.gradle @@ -0,0 +1,69 @@ +// Java plugin, toolchain (Java 17 / Adoptium), options.release, and repositories +// are inherited from the parent's subprojects {} block. + +// Minimum langchain4j version that ships the OpenAI Responses API +// (OpenAiResponsesChatModel / OpenAiResponsesStreamingChatModel, first released in 1.14.0). +def langchainVersion = '1.14.0' +// Test against a recent release to exercise forward compatibility (and match the version +// used to record btx cassettes). +def langchainTestVersion = '1.19.0' + +muzzle { + pass { + group = 'dev.langchain4j' + module = 'langchain4j' + versions = "[${langchainVersion},)" + extraDependency 'dev.langchain4j:langchain4j-http-client' + extraDependency 'dev.langchain4j:langchain4j-open-ai' + extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8' + } + // The Responses API classes this module targets did not exist before 1.14.0, so it must + // not apply to older releases (langchain_1_8_0 covers [1.8.0,1.14.0)). + fail { + group = 'dev.langchain4j' + module = 'langchain4j' + pinVersions '1.13.0' + extraDependency 'dev.langchain4j:langchain4j-http-client' + extraDependency 'dev.langchain4j:langchain4j-open-ai' + extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8' + } +} + +dependencies { + compileOnly project(':braintrust-java-agent:instrumenter') + implementation "io.opentelemetry:opentelemetry-api:${otelVersion}" + implementation 'com.google.code.findbugs:jsr305:3.0.2' // for @Nullable annotations + implementation "org.slf4j:slf4j-api:${slf4jVersion}" + implementation project(':braintrust-sdk') + + // ByteBuddy for ElementMatcher types used in instrumentation definitions + compileOnly 'net.bytebuddy:byte-buddy:1.17.5' + + // Target libraries — compileOnly because they will be on the app classpath at runtime + compileOnly "dev.langchain4j:langchain4j:${langchainVersion}" + compileOnly "dev.langchain4j:langchain4j-http-client:${langchainVersion}" + compileOnly "dev.langchain4j:langchain4j-open-ai:${langchainVersion}" + + // Test dependencies + testImplementation(testFixtures(project(":test-harness"))) + testImplementation project(':braintrust-java-agent:instrumenter') + testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + testImplementation 'net.bytebuddy:byte-buddy-agent:1.17.5' + testRuntimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}" + testImplementation "dev.langchain4j:langchain4j:${langchainTestVersion}" + testImplementation "dev.langchain4j:langchain4j-http-client:${langchainTestVersion}" + testImplementation "dev.langchain4j:langchain4j-open-ai:${langchainTestVersion}" +} + +test { + useJUnitPlatform() + workingDir = rootProject.projectDir + testLogging { + events "passed", "skipped", "failed" + showStandardStreams = true + exceptionFormat "full" + } +} diff --git a/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/BraintrustLangchain.java b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/BraintrustLangchain.java new file mode 100644 index 00000000..7a89bbde --- /dev/null +++ b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/BraintrustLangchain.java @@ -0,0 +1,242 @@ +package dev.braintrust.instrumentation.langchain.v1_14_0; + +import dev.langchain4j.model.openai.OpenAiChatModel; +import dev.langchain4j.model.openai.OpenAiResponsesChatModel; +import dev.langchain4j.model.openai.OpenAiResponsesStreamingChatModel; +import dev.langchain4j.model.openai.OpenAiStreamingChatModel; +import dev.langchain4j.service.AiServiceContext; +import dev.langchain4j.service.AiServices; +import dev.langchain4j.service.tool.ToolExecutor; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Tracer; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; + +/** Braintrust LangChain4j client instrumentation. */ +@Slf4j +public final class BraintrustLangchain { + + private static final String INSTRUMENTATION_NAME = "braintrust-langchain4j"; + private static final ThreadLocal AI_SERVICES_RECURSION_GUARD = + ThreadLocal.withInitial(() -> false); + + @SuppressWarnings("unchecked") + public static T wrap(OpenTelemetry openTelemetry, AiServices aiServices) { + if (AI_SERVICES_RECURSION_GUARD.get()) { + // already wrapped + return null; + } + AI_SERVICES_RECURSION_GUARD.set(true); + try { + AiServiceContext context = getPrivateField(aiServices, "context"); + Tracer tracer = openTelemetry.getTracer(INSTRUMENTATION_NAME); + + // ////// CREATE A LLM SPAN FOR EACH CALL TO AI PROVIDER + var chatModel = context.chatModel; + var streamingChatModel = context.streamingChatModel; + if (chatModel != null) { + if (chatModel instanceof OpenAiChatModel oaiModel) { + aiServices.chatModel(wrap(openTelemetry, oaiModel)); + } else { + log.warn( + "unsupported model: {}. LLM calls will not be instrumented", + chatModel.getClass().getName()); + } + // intentional fall-through + } else if (streamingChatModel != null) { + if (streamingChatModel instanceof OpenAiStreamingChatModel oaiModel) { + aiServices.streamingChatModel(wrap(openTelemetry, oaiModel)); + } else { + log.warn( + "unsupported model: {}. LLM calls will not be instrumented", + streamingChatModel.getClass().getName()); + } + // intentional fall-through + } else { + // langchain is going to fail to build. don't apply instrumentation. + throw new RuntimeException("model or chat model must be set"); + } + + if (context.toolService != null) { + // ////// CREATE A SPAN FOR EACH TOOL CALL + for (Map.Entry entry : + context.toolService.toolExecutors().entrySet()) { + String toolName = entry.getKey(); + ToolExecutor original = entry.getValue(); + entry.setValue(new TracingToolExecutor(original, toolName, tracer)); + } + + // ////// LINK SPANS ACROSS CONCURRENT TOOL CALLS + var underlyingExecutor = context.toolService.executor(); + if (underlyingExecutor != null) { + aiServices.executeToolsConcurrently( + new OtelContextPassingExecutor(underlyingExecutor)); + } + } + + // ////// CREATE A SPAN ON SERVICE METHOD INVOKE + T service = aiServices.build(); + Class serviceInterface = (Class) context.aiServiceClass; + return TracingProxy.create(serviceInterface, service, tracer); + } catch (Exception e) { + log.warn("failed to apply langchain AI services instrumentation", e); + return aiServices.build(); + } finally { + AI_SERVICES_RECURSION_GUARD.set(false); + } + } + + /** Instrument langchain openai chat model with braintrust traces */ + public static OpenAiChatModel wrap( + OpenTelemetry otel, OpenAiChatModel.OpenAiChatModelBuilder builder) { + return wrap(otel, builder.build()); + } + + public static OpenAiChatModel wrap(OpenTelemetry otel, OpenAiChatModel model) { + try { + // Get the internal OpenAiClient from the chat model + Object internalClient = getPrivateField(model, "client"); + + // Get the HttpClient from the internal client + dev.langchain4j.http.client.HttpClient httpClient = + getPrivateField(internalClient, "httpClient"); + + if (httpClient instanceof WrappedHttpClient) { + log.debug("model already instrumented. skipping: {}", httpClient.getClass()); + return model; + } + + // Wrap the HttpClient with our instrumented version + dev.langchain4j.http.client.HttpClient wrappedHttpClient = + new WrappedHttpClient(otel, httpClient, new Options("openai")); + + setPrivateField(internalClient, "httpClient", wrappedHttpClient); + + return model; + } catch (Exception e) { + log.warn("failed to instrument OpenAiChatModel", e); + return model; + } + } + + /** Instrument langchain openai chat model with braintrust traces */ + public static OpenAiStreamingChatModel wrap( + OpenTelemetry otel, OpenAiStreamingChatModel.OpenAiStreamingChatModelBuilder builder) { + return wrap(otel, builder.build()); + } + + public static OpenAiStreamingChatModel wrap( + OpenTelemetry otel, OpenAiStreamingChatModel model) { + try { + // Get the internal OpenAiClient from the streaming chat model + Object internalClient = getPrivateField(model, "client"); + + // Get the HttpClient from the internal client + dev.langchain4j.http.client.HttpClient httpClient = + getPrivateField(internalClient, "httpClient"); + + if (httpClient instanceof WrappedHttpClient) { + log.debug("model already instrumented. skipping: {}", httpClient.getClass()); + return model; + } + + // Wrap the HttpClient with our instrumented version + dev.langchain4j.http.client.HttpClient wrappedHttpClient = + new WrappedHttpClient(otel, httpClient, new Options("openai")); + + setPrivateField(internalClient, "httpClient", wrappedHttpClient); + + return model; + } catch (Exception e) { + log.warn("failed to instrument OpenAiStreamingChatModel", e); + return model; + } + } + + /** Instrument a langchain openai responses model with braintrust traces. */ + public static OpenAiResponsesChatModel wrap( + OpenTelemetry otel, OpenAiResponsesChatModel.Builder builder) { + return wrap(otel, builder.build()); + } + + public static OpenAiResponsesChatModel wrap( + OpenTelemetry otel, OpenAiResponsesChatModel model) { + wrapResponsesHttpClient(otel, model, "OpenAiResponsesChatModel"); + return model; + } + + /** Instrument a langchain openai streaming responses model with braintrust traces. */ + public static OpenAiResponsesStreamingChatModel wrap( + OpenTelemetry otel, OpenAiResponsesStreamingChatModel.Builder builder) { + return wrap(otel, builder.build()); + } + + public static OpenAiResponsesStreamingChatModel wrap( + OpenTelemetry otel, OpenAiResponsesStreamingChatModel model) { + wrapResponsesHttpClient(otel, model, "OpenAiResponsesStreamingChatModel"); + return model; + } + + /** + * Swaps the {@code httpClient} inside a responses model's internal {@code + * OpenAiResponsesClient} for an instrumented {@link WrappedHttpClient}. Both {@link + * OpenAiResponsesChatModel} and {@link OpenAiResponsesStreamingChatModel} hold an {@code + * OpenAiResponsesClient client} field with the same {@code + * dev.langchain4j.http.client.HttpClient httpClient} field as the regular chat models, so the + * tracing strategy is identical — the client POSTs to {@code /v1/responses} and {@code + * InstrumentationSemConv} tags the responses payload. + */ + private static void wrapResponsesHttpClient( + OpenTelemetry otel, Object model, String modelName) { + try { + Object internalClient = getPrivateField(model, "client"); + dev.langchain4j.http.client.HttpClient httpClient = + getPrivateField(internalClient, "httpClient"); + + if (httpClient instanceof WrappedHttpClient) { + log.debug("model already instrumented. skipping: {}", httpClient.getClass()); + return; + } + + dev.langchain4j.http.client.HttpClient wrappedHttpClient = + new WrappedHttpClient(otel, httpClient, new Options("openai")); + setPrivateField(internalClient, "httpClient", wrappedHttpClient); + } catch (Exception e) { + log.warn("failed to instrument {}", modelName, e); + } + } + + public record Options(String providerName) {} + + @SuppressWarnings("unchecked") + private static T getPrivateField(Object obj, String fieldName) + throws ReflectiveOperationException { + Class clazz = obj.getClass(); + while (clazz != null) { + try { + java.lang.reflect.Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + return (T) field.get(obj); + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); + } + + private static void setPrivateField(Object obj, String fieldName, Object value) + throws ReflectiveOperationException { + Class clazz = obj.getClass(); + while (clazz != null) { + try { + java.lang.reflect.Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(obj, value); + return; + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); + } +} diff --git a/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/OtelContextPassingExecutor.java b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/OtelContextPassingExecutor.java new file mode 100644 index 00000000..a36018c6 --- /dev/null +++ b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/OtelContextPassingExecutor.java @@ -0,0 +1,29 @@ +package dev.braintrust.instrumentation.langchain.v1_14_0; + +import io.opentelemetry.context.Context; +import java.util.concurrent.Executor; + +/** + * An executor that links open telemetry spans across threads. + * + *

    Any tasks submitted to the executor will point to the parent context that was present at the + * time of task submission. + */ +class OtelContextPassingExecutor implements Executor { + private final Executor underlying; + + public OtelContextPassingExecutor(Executor executor) { + this.underlying = executor; + } + + @Override + public void execute(Runnable command) { + var context = Context.current(); + underlying.execute( + () -> { + try (var ignored = context.makeCurrent()) { + command.run(); + } + }); + } +} diff --git a/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingProxy.java b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingProxy.java new file mode 100644 index 00000000..e889f154 --- /dev/null +++ b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingProxy.java @@ -0,0 +1,48 @@ +package dev.braintrust.instrumentation.langchain.v1_14_0; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; + +class TracingProxy { + /** + * Use a java {@link Proxy} to wrap a service interface methods with spans. + * + *

    Each interface method will create a span with the same name as the method. + */ + @SuppressWarnings("unchecked") + public static T create(Class serviceInterface, T service, Tracer tracer) { + return (T) + Proxy.newProxyInstance( + serviceInterface.getClassLoader(), + new Class[] {serviceInterface}, + (proxy, method, args) -> { + // Skip Object methods (equals, hashCode, toString) + if (method.getDeclaringClass() == Object.class) { + return method.invoke(service, args); + } + + Span span = tracer.spanBuilder(method.getName()).startSpan(); + try (Scope ignored = span.makeCurrent()) { + method.setAccessible(true); + return method.invoke(service, args); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + span.setStatus(StatusCode.ERROR, cause.getMessage()); + span.recordException(cause); + throw cause; + } catch (Exception e) { + span.setStatus(StatusCode.ERROR, e.getMessage()); + span.recordException(e); + throw e; + } finally { + span.end(); + } + }); + } + + private TracingProxy() {} +} diff --git a/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingToolExecutor.java b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingToolExecutor.java new file mode 100644 index 00000000..97cdadec --- /dev/null +++ b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingToolExecutor.java @@ -0,0 +1,78 @@ +package dev.braintrust.instrumentation.langchain.v1_14_0; + +import dev.langchain4j.agent.tool.ToolExecutionRequest; +import dev.langchain4j.invocation.InvocationContext; +import dev.langchain4j.service.tool.ToolExecutionResult; +import dev.langchain4j.service.tool.ToolExecutor; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +/** A ToolExecutor wrapper that creates a span around tool execution. */ +@Slf4j +class TracingToolExecutor implements ToolExecutor { + static final String TYPE_TOOL_JSON = "{\"type\":\"tool\"}"; + + private final ToolExecutor delegate; + private final String toolName; + private final Tracer tracer; + + TracingToolExecutor(ToolExecutor delegate, String toolName, Tracer tracer) { + this.delegate = delegate; + this.toolName = toolName; + this.tracer = tracer; + } + + @Override + public String execute(ToolExecutionRequest request, Object memoryId) { + Span span = tracer.spanBuilder(toolName).startSpan(); + try (Scope ignored = span.makeCurrent()) { + String result = delegate.execute(request, memoryId); + setSpanAttributes(span, request, result); + return result; + } catch (Exception e) { + span.setStatus(StatusCode.ERROR, e.getMessage()); + span.recordException(e); + throw e; + } finally { + span.end(); + } + } + + @Override + public ToolExecutionResult executeWithContext( + ToolExecutionRequest request, InvocationContext context) { + Span span = tracer.spanBuilder(toolName).startSpan(); + try (Scope ignored = span.makeCurrent()) { + ToolExecutionResult result = delegate.executeWithContext(request, context); + setSpanAttributes(span, request, result.resultText()); + return result; + } catch (Exception e) { + span.setStatus(StatusCode.ERROR, e.getMessage()); + span.recordException(e); + throw e; + } finally { + span.end(); + } + } + + private void setSpanAttributes( + Span span, ToolExecutionRequest request, @Nullable String toolCallResult) { + try { + span.setAttribute("braintrust.span_attributes", TYPE_TOOL_JSON); + + String args = request.arguments(); + if (args != null && !args.isEmpty()) { + span.setAttribute("braintrust.input_json", args); + } + if (toolCallResult != null) { + span.setAttribute("braintrust.output", toolCallResult); + } + } catch (Exception e) { + log.debug("Failed to set tool span attributes", e); + } + } +} diff --git a/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/WrappedHttpClient.java b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/WrappedHttpClient.java new file mode 100644 index 00000000..d6974a49 --- /dev/null +++ b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/WrappedHttpClient.java @@ -0,0 +1,219 @@ +package dev.braintrust.instrumentation.langchain.v1_14_0; + +import dev.braintrust.bootstrap.BraintrustBridge; +import dev.braintrust.instrumentation.InstrumentationSemConv; +import dev.braintrust.instrumentation.SseResponseAccumulator; +import dev.braintrust.json.BraintrustJsonMapper; +import dev.langchain4j.exception.HttpException; +import dev.langchain4j.http.client.HttpClient; +import dev.langchain4j.http.client.HttpRequest; +import dev.langchain4j.http.client.SuccessfulHttpResponse; +import dev.langchain4j.http.client.sse.ServerSentEvent; +import dev.langchain4j.http.client.sse.ServerSentEventContext; +import dev.langchain4j.http.client.sse.ServerSentEventListener; +import dev.langchain4j.http.client.sse.ServerSentEventParser; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import java.net.URI; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +class WrappedHttpClient implements HttpClient { + private final Tracer tracer; + private final HttpClient underlying; + private final BraintrustLangchain.Options options; + + public WrappedHttpClient( + OpenTelemetry openTelemetry, + HttpClient underlying, + BraintrustLangchain.Options options) { + this.tracer = openTelemetry.getTracer(BraintrustBridge.INSTRUMENTATION_NAME); + this.underlying = underlying; + this.options = options; + } + + @Override + public SuccessfulHttpResponse execute(HttpRequest request) + throws HttpException, RuntimeException { + Instant llmSpanStart = Instant.now(); + Span span = + tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME) + .setSpanKind(SpanKind.CLIENT) + .setStartTimestamp(llmSpanStart) + .startSpan(); + try (Scope scope = span.makeCurrent()) { + tagRequest(span, request); + var response = underlying.execute(request); + InstrumentationSemConv.tagLLMSpanResponse( + span, options.providerName(), response.body()); + InstrumentationSemConv.addServerSideChildSpans( + tracer, span, options.providerName(), response.body(), llmSpanStart); + return response; + } catch (Throwable t) { + InstrumentationSemConv.tagLLMSpanResponse(span, t); + throw t; + } finally { + span.end(); + } + } + + @Override + public void execute(HttpRequest request, ServerSentEventListener listener) { + if (listener instanceof WrappedServerSentEventListener) { + underlying.execute(request, listener); + return; + } + Instant llmSpanStart = Instant.now(); + Span span = + tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME) + .setSpanKind(SpanKind.CLIENT) + .setStartTimestamp(llmSpanStart) + .startSpan(); + try (Scope ignored = span.makeCurrent()) { + tagRequest(span, request); + underlying.execute( + request, + new WrappedServerSentEventListener( + listener, span, options.providerName(), tracer, llmSpanStart)); + } catch (Throwable t) { + InstrumentationSemConv.tagLLMSpanResponse(span, t); + span.end(); + throw t; + } + } + + @Override + public void execute( + HttpRequest request, ServerSentEventParser parser, ServerSentEventListener listener) { + if (listener instanceof WrappedServerSentEventListener) { + underlying.execute(request, parser, listener); + return; + } + Instant llmSpanStart = Instant.now(); + Span span = + tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME) + .setSpanKind(SpanKind.CLIENT) + .setStartTimestamp(llmSpanStart) + .startSpan(); + try (Scope ignored = span.makeCurrent()) { + tagRequest(span, request); + underlying.execute( + request, + parser, + new WrappedServerSentEventListener( + listener, span, options.providerName(), tracer, llmSpanStart)); + } catch (Throwable t) { + InstrumentationSemConv.tagLLMSpanResponse(span, t); + span.end(); + throw t; + } + } + + private void tagRequest(Span span, HttpRequest request) { + try { + URI uri = new URI(request.url()); + String baseUrl = uri.getScheme() + "://" + uri.getAuthority(); + List pathSegments = + Arrays.stream(uri.getPath().split("/")).filter(s -> !s.isEmpty()).toList(); + InstrumentationSemConv.tagLLMSpanRequest( + span, options.providerName(), baseUrl, pathSegments, "POST", request.body()); + } catch (Exception e) { + log.debug("Failed to tag request span", e); + } + } + + static class WrappedServerSentEventListener implements ServerSentEventListener { + private final ServerSentEventListener delegate; + private final Span span; + private final String providerName; + private final Tracer tracer; + private final Instant llmSpanStart; + private final long startNanos = System.nanoTime(); + private final AtomicLong timeToFirstTokenNanos = new AtomicLong(); + private final SseResponseAccumulator accumulator = + new SseResponseAccumulator(BraintrustJsonMapper.get()); + + WrappedServerSentEventListener( + ServerSentEventListener delegate, + Span span, + String providerName, + Tracer tracer, + Instant llmSpanStart) { + this.delegate = delegate; + this.span = span; + this.providerName = providerName; + this.tracer = tracer; + this.llmSpanStart = llmSpanStart; + } + + @Override + public void onOpen(SuccessfulHttpResponse response) { + try (Scope ignored = span.makeCurrent()) { + delegate.onOpen(response); + } + } + + @Override + public void onEvent(ServerSentEvent event, ServerSentEventContext context) { + try (Scope ignored = span.makeCurrent()) { + accumulateChunk(event.data()); + delegate.onEvent(event, context); + } + } + + @Override + public void onEvent(ServerSentEvent event) { + try (Scope ignored = span.makeCurrent()) { + accumulateChunk(event.data()); + delegate.onEvent(event); + } + } + + @Override + public void onError(Throwable error) { + try (Scope ignored = span.makeCurrent()) { + delegate.onError(error); + } finally { + InstrumentationSemConv.tagLLMSpanResponse(span, error); + span.end(); + } + } + + @Override + public void onClose() { + try (Scope ignored = span.makeCurrent()) { + delegate.onClose(); + } finally { + finalizeSpan(); + span.end(); + } + } + + private void accumulateChunk(String data) { + if (data == null || data.isEmpty() || "[DONE]".equals(data)) return; + if (timeToFirstTokenNanos.get() == 0L) { + timeToFirstTokenNanos.compareAndExchange(0L, System.nanoTime() - startNanos); + } + accumulator.merge(data); + } + + private void finalizeSpan() { + try { + Long ttft = timeToFirstTokenNanos.get(); + String responseBody = accumulator.build(); + InstrumentationSemConv.tagLLMSpanResponse(span, providerName, responseBody, ttft); + InstrumentationSemConv.addServerSideChildSpans( + tracer, span, providerName, responseBody, llmSpanStart); + } catch (Exception e) { + log.debug("Failed to finalize streaming span", e); + } + } + } +} diff --git a/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/WrappedHttpClientBuilder.java b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/WrappedHttpClientBuilder.java new file mode 100644 index 00000000..13b1b21c --- /dev/null +++ b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/WrappedHttpClientBuilder.java @@ -0,0 +1,48 @@ +package dev.braintrust.instrumentation.langchain.v1_14_0; + +import dev.langchain4j.http.client.HttpClient; +import dev.langchain4j.http.client.HttpClientBuilder; +import io.opentelemetry.api.OpenTelemetry; +import java.time.Duration; + +class WrappedHttpClientBuilder implements HttpClientBuilder { + private final OpenTelemetry openTelemetry; + private final HttpClientBuilder underlying; + private final BraintrustLangchain.Options options; + + public WrappedHttpClientBuilder( + OpenTelemetry openTelemetry, + HttpClientBuilder underlying, + BraintrustLangchain.Options options) { + this.openTelemetry = openTelemetry; + this.underlying = underlying; + this.options = options; + } + + @Override + public Duration connectTimeout() { + return underlying.connectTimeout(); + } + + @Override + public HttpClientBuilder connectTimeout(Duration timeout) { + underlying.connectTimeout(timeout); + return this; + } + + @Override + public Duration readTimeout() { + return underlying.readTimeout(); + } + + @Override + public HttpClientBuilder readTimeout(Duration timeout) { + underlying.readTimeout(timeout); + return this; + } + + @Override + public HttpClient build() { + return new WrappedHttpClient(openTelemetry, underlying.build(), options); + } +} diff --git a/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/auto/LangchainInstrumentationModule.java b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/auto/LangchainInstrumentationModule.java new file mode 100644 index 00000000..072c9747 --- /dev/null +++ b/braintrust-sdk/instrumentation/langchain_1_14_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_14_0/auto/LangchainInstrumentationModule.java @@ -0,0 +1,228 @@ +package dev.braintrust.instrumentation.langchain.v1_14_0.auto; + +import static net.bytebuddy.matcher.ElementMatchers.*; + +import com.google.auto.service.AutoService; +import dev.braintrust.instrumentation.InstrumentationModule; +import dev.braintrust.instrumentation.TypeInstrumentation; +import dev.braintrust.instrumentation.TypeTransformer; +import dev.braintrust.instrumentation.langchain.v1_14_0.BraintrustLangchain; +import dev.braintrust.instrumentation.muzzle.ClassLoaderMatchers; +import dev.langchain4j.model.openai.OpenAiChatModel; +import dev.langchain4j.model.openai.OpenAiResponsesChatModel; +import dev.langchain4j.model.openai.OpenAiResponsesStreamingChatModel; +import dev.langchain4j.model.openai.OpenAiStreamingChatModel; +import dev.langchain4j.service.AiServices; +import io.opentelemetry.api.GlobalOpenTelemetry; +import java.util.List; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.implementation.bytecode.assign.Assigner; +import net.bytebuddy.matcher.ElementMatcher; + +@AutoService(InstrumentationModule.class) +public class LangchainInstrumentationModule extends InstrumentationModule { + private static final String MANUAL_PACKAGE = + "dev.braintrust.instrumentation.langchain.v1_14_0."; + + public LangchainInstrumentationModule() { + super("langchain_1_14_0"); + } + + /** + * Gates this module to langchain4j >= 1.14.0, where the OpenAI Responses API classes ({@code + * OpenAiResponsesChatModel} et al.) first appeared. Earlier releases are covered by the {@code + * langchain_1_8_0} module, whose matcher excludes 1.14.0+ — so exactly one module applies for + * any given langchain4j version and the two never overlap. + */ + @Override + public ElementMatcher classLoaderMatcher() { + return ClassLoaderMatchers.hasClassNamed( + "dev.langchain4j.model.openai.OpenAiResponsesChatModel"); + } + + @Override + public List getHelperClassNames() { + return List.of( + MANUAL_PACKAGE + "BraintrustLangchain", + MANUAL_PACKAGE + "BraintrustLangchain$Options", + MANUAL_PACKAGE + "WrappedHttpClient", + MANUAL_PACKAGE + "WrappedHttpClient$WrappedServerSentEventListener", + MANUAL_PACKAGE + "WrappedHttpClientBuilder", + MANUAL_PACKAGE + "TracingProxy", + MANUAL_PACKAGE + "TracingToolExecutor", + MANUAL_PACKAGE + "OtelContextPassingExecutor", + "dev.braintrust.instrumentation.SseResponseAccumulator", + "dev.braintrust.instrumentation.InstrumentationSemConv", + "dev.braintrust.json.BraintrustJsonMapper"); + } + + @Override + public List typeInstrumentations() { + return List.of( + new OpenAiChatModelBuilderInstrumentation(), + new OpenAiStreamingChatModelBuilderInstrumentation(), + new OpenAiResponsesChatModelBuilderInstrumentation(), + new OpenAiResponsesStreamingChatModelBuilderInstrumentation(), + new AiServicesInstrumentation()); + } + + // ------------------------------------------------------------------------- + // Intercept OpenAiChatModel.Builder.build() to wrap the HTTP client + // ------------------------------------------------------------------------- + + public static class OpenAiChatModelBuilderInstrumentation implements TypeInstrumentation { + @Override + public ElementMatcher typeMatcher() { + return named("dev.langchain4j.model.openai.OpenAiChatModel$OpenAiChatModelBuilder"); + } + + @Override + public void transform(TypeTransformer transformer) { + transformer.applyAdviceToMethod( + named("build").and(takesArguments(0)), + LangchainInstrumentationModule.class.getName() + + "$OpenAiChatModelBuilderAdvice"); + } + } + + private static class OpenAiChatModelBuilderAdvice { + @Advice.OnMethodExit + public static void build( + @Advice.Return(readOnly = false, typing = Assigner.Typing.DYNAMIC) + Object returnedModel) { + returnedModel = + BraintrustLangchain.wrap( + GlobalOpenTelemetry.get(), (OpenAiChatModel) returnedModel); + } + } + + // ------------------------------------------------------------------------- + // Intercept OpenAiStreamingChatModel.Builder.build() to wrap the HTTP client + // ------------------------------------------------------------------------- + + public static class OpenAiStreamingChatModelBuilderInstrumentation + implements TypeInstrumentation { + @Override + public ElementMatcher typeMatcher() { + return named( + "dev.langchain4j.model.openai.OpenAiStreamingChatModel$OpenAiStreamingChatModelBuilder"); + } + + @Override + public void transform(TypeTransformer transformer) { + transformer.applyAdviceToMethod( + named("build").and(takesArguments(0)), + LangchainInstrumentationModule.class.getName() + + "$OpenAiStreamingChatModelBuilderAdvice"); + } + } + + private static class OpenAiStreamingChatModelBuilderAdvice { + @Advice.OnMethodExit + public static void build( + @Advice.Return(readOnly = false, typing = Assigner.Typing.DYNAMIC) + Object returnedModel) { + returnedModel = + BraintrustLangchain.wrap( + GlobalOpenTelemetry.get(), (OpenAiStreamingChatModel) returnedModel); + } + } + + // ------------------------------------------------------------------------- + // Intercept OpenAiResponsesChatModel.Builder.build() to wrap the HTTP client + // ------------------------------------------------------------------------- + + public static class OpenAiResponsesChatModelBuilderInstrumentation + implements TypeInstrumentation { + @Override + public ElementMatcher typeMatcher() { + return named("dev.langchain4j.model.openai.OpenAiResponsesChatModel$Builder"); + } + + @Override + public void transform(TypeTransformer transformer) { + transformer.applyAdviceToMethod( + named("build").and(takesArguments(0)), + LangchainInstrumentationModule.class.getName() + + "$OpenAiResponsesChatModelBuilderAdvice"); + } + } + + private static class OpenAiResponsesChatModelBuilderAdvice { + @Advice.OnMethodExit + public static void build( + @Advice.Return(readOnly = false, typing = Assigner.Typing.DYNAMIC) + Object returnedModel) { + returnedModel = + BraintrustLangchain.wrap( + GlobalOpenTelemetry.get(), (OpenAiResponsesChatModel) returnedModel); + } + } + + // ------------------------------------------------------------------------- + // Intercept OpenAiResponsesStreamingChatModel.Builder.build() to wrap the HTTP client + // ------------------------------------------------------------------------- + + public static class OpenAiResponsesStreamingChatModelBuilderInstrumentation + implements TypeInstrumentation { + @Override + public ElementMatcher typeMatcher() { + return named("dev.langchain4j.model.openai.OpenAiResponsesStreamingChatModel$Builder"); + } + + @Override + public void transform(TypeTransformer transformer) { + transformer.applyAdviceToMethod( + named("build").and(takesArguments(0)), + LangchainInstrumentationModule.class.getName() + + "$OpenAiResponsesStreamingChatModelBuilderAdvice"); + } + } + + private static class OpenAiResponsesStreamingChatModelBuilderAdvice { + @Advice.OnMethodExit + public static void build( + @Advice.Return(readOnly = false, typing = Assigner.Typing.DYNAMIC) + Object returnedModel) { + returnedModel = + BraintrustLangchain.wrap( + GlobalOpenTelemetry.get(), + (OpenAiResponsesStreamingChatModel) returnedModel); + } + } + + // ------------------------------------------------------------------------ - + // Intercept AiServices.build() to wrap with TracingProxy + TracingToolExecutor + // ------------------------------------------------------------------------- + + public static class AiServicesInstrumentation implements TypeInstrumentation { + @Override + public ElementMatcher typeMatcher() { + return hasSuperType(named("dev.langchain4j.service.AiServices")) + .and( + declaresMethod( + named("build").and(takesArguments(0)).and(not(isAbstract())))); + } + + @Override + public void transform(TypeTransformer transformer) { + transformer.applyAdviceToMethod( + named("build").and(takesArguments(0)), + LangchainInstrumentationModule.class.getName() + "$AiServicesAdvice"); + } + } + + private static class AiServicesAdvice { + @Advice.OnMethodExit + public static void build( + @Advice.This AiServices aiServices, + @Advice.Return(readOnly = false, typing = Assigner.Typing.DYNAMIC) + Object returnedService) { + var wrapped = BraintrustLangchain.wrap(GlobalOpenTelemetry.get(), aiServices); + if (wrapped != null) { + returnedService = wrapped; + } + } + } +} diff --git a/braintrust-sdk/instrumentation/langchain_1_14_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_14_0/BraintrustLangchainTest.java b/braintrust-sdk/instrumentation/langchain_1_14_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_14_0/BraintrustLangchainTest.java new file mode 100644 index 00000000..00a338d0 --- /dev/null +++ b/braintrust-sdk/instrumentation/langchain_1_14_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_14_0/BraintrustLangchainTest.java @@ -0,0 +1,498 @@ +package dev.braintrust.instrumentation.langchain.v1_14_0; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.TestHarness; +import dev.braintrust.instrumentation.Instrumenter; +import dev.langchain4j.agent.tool.Tool; +import dev.langchain4j.agent.tool.ToolSpecification; +import dev.langchain4j.data.message.UserMessage; +import dev.langchain4j.model.chat.ChatModel; +import dev.langchain4j.model.chat.StreamingChatModel; +import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.request.json.JsonObjectSchema; +import dev.langchain4j.model.chat.response.ChatResponse; +import dev.langchain4j.model.chat.response.StreamingChatResponseHandler; +import dev.langchain4j.model.openai.OpenAiChatModel; +import dev.langchain4j.model.openai.OpenAiResponsesChatModel; +import dev.langchain4j.model.openai.OpenAiStreamingChatModel; +import dev.langchain4j.service.AiServices; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.sdk.trace.data.SpanData; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.SneakyThrows; +import net.bytebuddy.agent.ByteBuddyAgent; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class BraintrustLangchainTest { + + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + + @BeforeAll + public static void beforeAll() { + var instrumentation = ByteBuddyAgent.install(); + Instrumenter.install(instrumentation, BraintrustLangchainTest.class.getClassLoader()); + } + + private TestHarness testHarness; + + @BeforeEach + void beforeEach() { + testHarness = TestHarness.setup(); + } + + @Test + @SneakyThrows + void testSyncChatCompletion() { + ChatModel model = + OpenAiChatModel.builder() + .apiKey(testHarness.openAiApiKey()) + .baseUrl(testHarness.openAiBaseUrl()) + .modelName("gpt-4o-mini") + .temperature(0.0) + .build(); + + var message = UserMessage.from("What is the capital of France?"); + var response = model.chat(message); + + assertNotNull(response); + assertNotNull(response.aiMessage().text()); + + var spans = testHarness.awaitExportedSpans(); + assertEquals(1, spans.size(), "Expected one span for sync chat completion"); + var span = spans.get(0); + + assertEquals("Chat Completion", span.getName(), "Span name should be 'Chat Completion'"); + + var attributes = span.getAttributes(); + var braintrustSpanAttributesJson = + attributes.get(AttributeKey.stringKey("braintrust.span_attributes")); + + JsonNode spanAttributes = JSON_MAPPER.readTree(braintrustSpanAttributesJson); + assertEquals("llm", spanAttributes.get("type").asText(), "Span type should be 'llm'"); + + String metadataJson = attributes.get(AttributeKey.stringKey("braintrust.metadata")); + assertNotNull(metadataJson, "Metadata should be present"); + JsonNode metadata = JSON_MAPPER.readTree(metadataJson); + assertEquals("openai", metadata.get("provider").asText(), "Provider should be 'openai'"); + assertEquals( + "gpt-4o-mini", metadata.get("model").asText(), "Model should be 'gpt-4o-mini'"); + + String metricsJson = attributes.get(AttributeKey.stringKey("braintrust.metrics")); + assertNotNull(metricsJson, "Metrics should be present"); + JsonNode metrics = JSON_MAPPER.readTree(metricsJson); + assertTrue(metrics.get("tokens").asLong() > 0, "Total tokens should be > 0"); + assertTrue(metrics.get("prompt_tokens").asLong() > 0, "Prompt tokens should be > 0"); + assertTrue( + metrics.get("completion_tokens").asLong() > 0, "Completion tokens should be > 0"); + assertFalse( + metrics.has("time_to_first_token"), + "time_to_first_token should not be present for non-streaming"); + + String inputJson = attributes.get(AttributeKey.stringKey("braintrust.input_json")); + assertNotNull(inputJson, "Input should be present"); + JsonNode input = JSON_MAPPER.readTree(inputJson); + assertTrue(input.isArray(), "Input should be an array"); + assertTrue(input.size() > 0, "Input array should not be empty"); + assertTrue( + input.get(0).get("content").asText().contains("What is the capital of France"), + "Input should contain the user message"); + + String outputJson = attributes.get(AttributeKey.stringKey("braintrust.output_json")); + assertNotNull(outputJson, "Output should be present"); + JsonNode output = JSON_MAPPER.readTree(outputJson); + assertTrue(output.isArray(), "Output should be an array"); + assertTrue(output.size() > 0, "Output array should not be empty"); + assertNotNull( + output.get(0).get("message").get("content"), + "Output should contain assistant response content"); + + // The serialized span output should reflect the full response the client received. + assertSpanOutputReflects(response, span); + } + + /** + * Exercises the OpenAI Responses API path (OpenAiResponsesChatModel -> /v1/responses), which is + * only available on this module's langchain4j range (>= 1.14.0). Auto-instrumentation wraps the + * responses model's HTTP client on build(); a single llm span should be produced and tagged. + */ + @Test + @SneakyThrows + void testResponsesApi() { + ChatModel model = + OpenAiResponsesChatModel.builder() + .apiKey(testHarness.openAiApiKey()) + .baseUrl(testHarness.openAiBaseUrl()) + .modelName("gpt-4o-mini") + .build(); + + var response = model.chat(UserMessage.from("What is the capital of France?")); + assertNotNull(response); + assertNotNull(response.aiMessage().text()); + + var spans = testHarness.awaitExportedSpans(); + assertEquals(1, spans.size(), "Expected one span for a responses-API call"); + var span = spans.get(0); + assertEquals("responses", span.getName(), "Span name should be 'responses'"); + + var attributes = span.getAttributes(); + JsonNode spanAttributes = + JSON_MAPPER.readTree( + attributes.get(AttributeKey.stringKey("braintrust.span_attributes"))); + assertEquals("llm", spanAttributes.get("type").asText(), "Span type should be 'llm'"); + + JsonNode metadata = + JSON_MAPPER.readTree(attributes.get(AttributeKey.stringKey("braintrust.metadata"))); + assertEquals("openai", metadata.get("provider").asText(), "Provider should be 'openai'"); + + String metricsJson = attributes.get(AttributeKey.stringKey("braintrust.metrics")); + assertNotNull(metricsJson, "Metrics should be present"); + JsonNode metrics = JSON_MAPPER.readTree(metricsJson); + assertTrue(metrics.get("tokens").asLong() > 0, "Total tokens should be > 0"); + + assertNotNull( + attributes.get(AttributeKey.stringKey("braintrust.input_json")), + "Input should be present"); + assertNotNull( + attributes.get(AttributeKey.stringKey("braintrust.output_json")), + "Output should be present"); + } + + @Test + @SneakyThrows + void testStreamingChatCompletion() { + var tracer = testHarness.openTelemetry().getTracer("test-tracer"); + + // Auto-instrumentation intercepts OpenAiStreamingChatModel.Builder.build() + StreamingChatModel model = + OpenAiStreamingChatModel.builder() + .apiKey(testHarness.openAiApiKey()) + .baseUrl(testHarness.openAiBaseUrl()) + .modelName("gpt-4o-mini") + .temperature(0.0) + .build(); + + var future = new CompletableFuture(); + var responseBuilder = new StringBuilder(); + var callbackCount = new AtomicInteger(0); + + model.chat( + "What is the capital of France?", + new StreamingChatResponseHandler() { + @Override + public void onPartialResponse(String token) { + Span childSpan = + tracer.spanBuilder( + "callback-span-" + callbackCount.incrementAndGet()) + .startSpan(); + childSpan.end(); + responseBuilder.append(token); + } + + @Override + public void onCompleteResponse(ChatResponse response) { + future.complete(response); + } + + @Override + public void onError(Throwable error) { + future.completeExceptionally(error); + } + }); + + var response = future.get(); + + assertNotNull(response); + assertFalse(responseBuilder.toString().isEmpty(), "Response should not be empty"); + + int expectedMinSpans = 1 + callbackCount.get(); + var spans = testHarness.awaitExportedSpans(expectedMinSpans); + assertTrue( + spans.size() >= expectedMinSpans, + "Expected at least " + expectedMinSpans + " spans, got " + spans.size()); + + SpanData llmSpan = null; + List callbackSpans = new java.util.ArrayList<>(); + + for (var span : spans) { + if (span.getName().equals("Chat Completion")) { + llmSpan = span; + } else if (span.getName().startsWith("callback-span-")) { + callbackSpans.add(span); + } + } + + assertNotNull(llmSpan, "Should have an LLM span named 'Chat Completion'"); + assertEquals( + callbackCount.get(), + callbackSpans.size(), + "Should have one callback span per onPartialResponse invocation"); + + String llmSpanId = llmSpan.getSpanId(); + for (var callbackSpan : callbackSpans) { + assertEquals( + llmSpanId, + callbackSpan.getParentSpanId(), + "Callback span '" + + callbackSpan.getName() + + "' should be parented under LLM span"); + } + + var attributes = llmSpan.getAttributes(); + + var braintrustSpanAttributesJson = + attributes.get(AttributeKey.stringKey("braintrust.span_attributes")); + + JsonNode spanAttributes = JSON_MAPPER.readTree(braintrustSpanAttributesJson); + assertEquals("llm", spanAttributes.get("type").asText(), "Span type should be 'llm'"); + + String metadataJson = attributes.get(AttributeKey.stringKey("braintrust.metadata")); + assertNotNull(metadataJson, "Metadata should be present"); + JsonNode metadata = JSON_MAPPER.readTree(metadataJson); + assertEquals("openai", metadata.get("provider").asText(), "Provider should be 'openai'"); + assertEquals( + "gpt-4o-mini", metadata.get("model").asText(), "Model should be 'gpt-4o-mini'"); + + String metricsJson = attributes.get(AttributeKey.stringKey("braintrust.metrics")); + assertNotNull(metricsJson, "Metrics should be present"); + JsonNode metrics = JSON_MAPPER.readTree(metricsJson); + assertTrue(metrics.get("tokens").asLong() > 0, "Total tokens should be > 0"); + assertTrue(metrics.get("prompt_tokens").asLong() > 0, "Prompt tokens should be > 0"); + assertTrue( + metrics.get("completion_tokens").asLong() > 0, "Completion tokens should be > 0"); + assertTrue( + metrics.has("time_to_first_token"), + "Metrics should contain time_to_first_token for streaming"); + assertTrue( + metrics.get("time_to_first_token").isNumber(), + "time_to_first_token should be a number"); + + String inputJson = attributes.get(AttributeKey.stringKey("braintrust.input_json")); + assertNotNull(inputJson, "Input should be present"); + JsonNode input = JSON_MAPPER.readTree(inputJson); + assertTrue(input.isArray(), "Input should be an array"); + assertTrue(input.size() > 0, "Input array should not be empty"); + assertTrue( + input.get(0).get("content").asText().contains("What is the capital of France"), + "Input should contain the user message"); + + String outputJson = attributes.get(AttributeKey.stringKey("braintrust.output_json")); + assertNotNull(outputJson, "Output should be present"); + JsonNode output = JSON_MAPPER.readTree(outputJson); + assertTrue(output.isArray(), "Output should be an array"); + assertTrue(output.size() > 0, "Output array should not be empty"); + JsonNode choice = output.get(0); + assertNotNull( + choice.get("message").get("content"), + "Output should contain the complete streamed response"); + assertNotNull(choice.get("finish_reason"), "Output should have finish_reason"); + + // The reconstructed streaming span output should reflect the full response the client + // received — the instrumentation must feed every SSE event to the accumulator. + assertSpanOutputReflects(response, llmSpan); + } + + @Test + @SneakyThrows + void testStreamingChatCompletionWithTools() { + // Auto-instrumentation intercepts OpenAiStreamingChatModel.Builder.build() + StreamingChatModel model = + OpenAiStreamingChatModel.builder() + .apiKey(testHarness.openAiApiKey()) + .baseUrl(testHarness.openAiBaseUrl()) + .modelName("gpt-4o") + .temperature(0.0) + .build(); + + var weatherTool = + ToolSpecification.builder() + .name("get_weather") + .description("Get the current weather for a location") + .parameters( + JsonObjectSchema.builder() + .addStringProperty( + "location", + "The city and state, e.g. San" + " Francisco, CA") + .required("location") + .build()) + .build(); + + var chatRequest = + ChatRequest.builder() + .messages(UserMessage.from("What is the weather in Paris, France?")) + .toolSpecifications(weatherTool) + .build(); + + var future = new CompletableFuture(); + model.chat( + chatRequest, + new StreamingChatResponseHandler() { + @Override + public void onPartialResponse(String token) {} + + @Override + public void onCompleteResponse(ChatResponse response) { + future.complete(response); + } + + @Override + public void onError(Throwable error) { + future.completeExceptionally(error); + } + }); + var response = future.get(); + + // The stream must carry tool-call deltas (merged by index) all the way to the span — the + // original bug dropped tool_calls entirely from streaming reconstruction. + assertTrue( + response.aiMessage().hasToolExecutionRequests(), + "Model should have requested a tool call"); + + var llmSpan = + testHarness.awaitExportedSpans(1).stream() + .filter(s -> s.getName().equals("Chat Completion")) + .findFirst() + .orElseThrow(() -> new AssertionError("no 'Chat Completion' llm span")); + + assertSpanOutputReflects(response, llmSpan); + } + + /** + * Asserts that the llm span's serialized output ({@code braintrust.output_json}) reflects the + * full response the langchain client received — comparing the reconstructed assistant message + * against the client's parsed {@link ChatResponse} (content, thinking, and tool calls) rather + * than hand-asserting individual fields per test. langchain decodes the same stream + * independently of our accumulator, so agreement is a meaningful end-to-end check. + */ + @SneakyThrows + private void assertSpanOutputReflects(ChatResponse clientResponse, SpanData llmSpan) { + String outputJson = + llmSpan.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); + assertNotNull(outputJson, "Span should have braintrust.output_json"); + JsonNode message = JSON_MAPPER.readTree(outputJson).get(0).get("message"); + assertNotNull(message, "Span output should contain a choice message"); + + var aiMessage = clientResponse.aiMessage(); + + if (aiMessage.text() != null) { + assertEquals( + aiMessage.text(), + message.path("content").asText(), + "Span output content should match the client's assistant text"); + } + if (aiMessage.thinking() != null) { + assertEquals( + aiMessage.thinking(), + message.path("reasoning_content").asText(), + "Span output reasoning_content should match the client's thinking"); + } + if (aiMessage.hasToolExecutionRequests()) { + JsonNode toolCalls = message.get("tool_calls"); + assertNotNull(toolCalls, "Span output should contain tool_calls"); + var requests = aiMessage.toolExecutionRequests(); + assertEquals( + requests.size(), toolCalls.size(), "tool_calls count should match the client"); + for (int i = 0; i < requests.size(); i++) { + var request = requests.get(i); + JsonNode function = toolCalls.get(i).get("function"); + assertEquals( + request.name(), function.get("name").asText(), "tool name should match"); + assertEquals( + JSON_MAPPER.readTree(request.arguments()), + JSON_MAPPER.readTree(function.get("arguments").asText()), + "tool arguments should match"); + if (request.id() != null) { + assertEquals( + request.id(), + toolCalls.get(i).get("id").asText(), + "tool id should match"); + } + } + } + } + + @Test + @SneakyThrows + void testAiServicesWithTools() { + // Auto-instrumentation intercepts both OpenAiChatModel.Builder.build() and + // AiServices.build() + Assistant assistant = + AiServices.builder(Assistant.class) + .chatModel( + OpenAiChatModel.builder() + .apiKey(testHarness.openAiApiKey()) + .baseUrl(testHarness.openAiBaseUrl()) + .modelName("gpt-4o-mini") + .temperature(0.0) + .build()) + .tools(new WeatherTools()) + .executeToolsConcurrently() + .build(); + + var response = assistant.chat("is it hotter in Paris or New York right now?"); + + assertNotNull(response); + + var spans = testHarness.awaitExportedSpans(3); + assertTrue(spans.size() >= 3, "Expected at least 3 spans for AI Services with tools"); + + int numServiceMethodSpans = 0; + int numLLMSpans = 0; + int numToolCallSpans = 0; + + for (var span : spans) { + String spanName = span.getName(); + var attributes = span.getAttributes(); + + if (spanName.equals("chat")) { + numServiceMethodSpans++; + } else if (spanName.equals("Chat Completion")) { + numLLMSpans++; + var spanAttributesJson = + attributes.get(AttributeKey.stringKey("braintrust.span_attributes")); + assertNotNull(spanAttributesJson, "LLM span should have span_attributes"); + JsonNode spanAttributes = JSON_MAPPER.readTree(spanAttributesJson); + assertEquals( + "llm", spanAttributes.get("type").asText(), "Span type should be 'llm'"); + } else if (spanName.equals("getWeather")) { + numToolCallSpans++; + var spanAttributesJson = + attributes.get(AttributeKey.stringKey("braintrust.span_attributes")); + assertNotNull(spanAttributesJson, "Tool span should have span_attributes"); + JsonNode spanAttributes = JSON_MAPPER.readTree(spanAttributesJson); + assertEquals( + "tool", spanAttributes.get("type").asText(), "Span type should be 'tool'"); + } + } + assertEquals(1, numServiceMethodSpans, "should be exactly one service call"); + assertTrue(numLLMSpans >= 2, "should be at least two llm spans"); + assertTrue(numToolCallSpans >= 2, "should be at least two tool call spans"); + } + + /** AI Service interface for the assistant */ + interface Assistant { + String chat(String userMessage); + } + + /** Example tool class with weather-related methods */ + public static class WeatherTools { + @Tool("Get current weather for a location") + public String getWeather(String location) { + return String.format("The weather in %s is sunny with 72°F temperature.", location); + } + + @Tool("Get weather forecast for next N days") + public String getForecast(String location, int days) { + return String.format( + "The %d-day forecast for %s: Mostly sunny with temperatures between 65-75°F.", + days, location); + } + } +} diff --git a/braintrust-sdk/instrumentation/langchain_1_14_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingToolExecutorTest.java b/braintrust-sdk/instrumentation/langchain_1_14_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingToolExecutorTest.java new file mode 100644 index 00000000..b192a20f --- /dev/null +++ b/braintrust-sdk/instrumentation/langchain_1_14_0/src/test/java/dev/braintrust/instrumentation/langchain/v1_14_0/TracingToolExecutorTest.java @@ -0,0 +1,18 @@ +package dev.braintrust.instrumentation.langchain.v1_14_0; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import dev.braintrust.json.BraintrustJsonMapper; +import java.util.Map; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; + +public class TracingToolExecutorTest { + @Test + @SneakyThrows + void typeToolJsonCorrect() { + assertEquals( + BraintrustJsonMapper.toJson(Map.of("type", "tool")), + TracingToolExecutor.TYPE_TOOL_JSON); + } +} diff --git a/braintrust-sdk/instrumentation/langchain_1_8_0/build.gradle b/braintrust-sdk/instrumentation/langchain_1_8_0/build.gradle index 74db5342..91481c08 100644 --- a/braintrust-sdk/instrumentation/langchain_1_8_0/build.gradle +++ b/braintrust-sdk/instrumentation/langchain_1_8_0/build.gradle @@ -4,10 +4,23 @@ def langchainVersion = '1.8.0' muzzle { + // Upper-bounded below 1.14.0: from 1.14.0 on the OpenAI Responses API exists and the + // langchain_1_14_0 module takes over (it also covers chat completions). pass { group = 'dev.langchain4j' module = 'langchain4j' - versions = '[1.8.0,)' + versions = '[1.8.0,1.14.0)' + extraDependency 'dev.langchain4j:langchain4j-http-client' + extraDependency 'dev.langchain4j:langchain4j-open-ai' + extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8' + } + // Assert the classLoaderMatcher rejects 1.14.0+ so the two langchain modules never both + // instrument OpenAiChatModel on the same classloader. + fail { + group = 'dev.langchain4j' + module = 'langchain4j' + pinVersions '1.14.0' extraDependency 'dev.langchain4j:langchain4j-http-client' extraDependency 'dev.langchain4j:langchain4j-open-ai' extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' diff --git a/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/auto/LangchainInstrumentationModule.java b/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/auto/LangchainInstrumentationModule.java index 63ddc6a0..80cdd172 100644 --- a/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/auto/LangchainInstrumentationModule.java +++ b/braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/auto/LangchainInstrumentationModule.java @@ -7,6 +7,7 @@ import dev.braintrust.instrumentation.TypeInstrumentation; import dev.braintrust.instrumentation.TypeTransformer; import dev.braintrust.instrumentation.langchain.v1_8_0.BraintrustLangchain; +import dev.braintrust.instrumentation.muzzle.ClassLoaderMatchers; import dev.langchain4j.model.openai.OpenAiChatModel; import dev.langchain4j.model.openai.OpenAiStreamingChatModel; import dev.langchain4j.service.AiServices; @@ -25,6 +26,20 @@ public LangchainInstrumentationModule() { super("langchain_1_8_0"); } + /** + * Gates this module to langchain4j < 1.14.0. The Responses API classes ({@code + * OpenAiResponsesChatModel} et al.) first appeared in 1.14.0; from there on the {@code + * langchain_1_14_0} module takes over (chat completions + responses). Excluding classloaders + * that already have the responses classes keeps the two modules from both instrumenting {@code + * OpenAiChatModel} on 1.14.0+. + */ + @Override + public ElementMatcher classLoaderMatcher() { + return not( + ClassLoaderMatchers.hasClassNamed( + "dev.langchain4j.model.openai.OpenAiResponsesChatModel")); + } + @Override public List getHelperClassNames() { return List.of( diff --git a/btx/build.gradle b/btx/build.gradle index b72303de..c60e95c1 100644 --- a/btx/build.gradle +++ b/btx/build.gradle @@ -43,7 +43,7 @@ dependencies { testImplementation project(':braintrust-sdk:instrumentation:openai_2_15_0') testImplementation project(':braintrust-sdk:instrumentation:anthropic_2_2_0') testImplementation project(':braintrust-sdk:instrumentation:genai_1_18_0') - testImplementation project(':braintrust-sdk:instrumentation:langchain_1_8_0') + testImplementation project(':braintrust-sdk:instrumentation:langchain_1_14_0') testImplementation project(':braintrust-sdk:instrumentation:springai_1_0_0') testImplementation project(':braintrust-sdk:instrumentation:aws_bedrock_2_30_0') @@ -70,10 +70,11 @@ dependencies { testRuntimeOnly 'io.projectreactor.netty:reactor-netty-http:1.2.3' testImplementation 'org.apache.httpcomponents.client5:httpclient5:5.3.1' - // LangChain4j - testImplementation 'dev.langchain4j:langchain4j:1.9.1' - testImplementation 'dev.langchain4j:langchain4j-http-client:1.9.1' - testImplementation 'dev.langchain4j:langchain4j-open-ai:1.9.1' + // LangChain4j — 1.14.0+ ships the OpenAI Responses API (OpenAiResponsesChatModel), which the + // langchain_1_14_0 module instruments. Pinned to a recent 1.x for the responses spec coverage. + testImplementation 'dev.langchain4j:langchain4j:1.19.0' + testImplementation 'dev.langchain4j:langchain4j-http-client:1.19.0' + testImplementation 'dev.langchain4j:langchain4j-open-ai:1.19.0' // OpenTelemetry testImplementation 'io.opentelemetry:opentelemetry-api:1.54.1' diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java index 387ed177..379d7e3d 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java @@ -130,7 +130,14 @@ static void validateValue(Object actual, Object expected, String context) { } else { // scalar: null expected means "don't care" if (expected == null) return; - if (!valuesEqual(actual, expected)) { + // A message's text content is legitimately represented either as a plain string or, + // in the OpenAI Responses API, as a single text content part + // ([{type: input_text|output_text|text, text: "..."}]). When the spec asserts the + // string form but an SDK (e.g. langchain4j's OpenAiResponsesChatModel) emits the + // content-part form, collapse it so the two representations compare equal. + Object normalizedActual = + expected instanceof String ? collapseTextContentParts(actual) : actual; + if (!valuesEqual(normalizedActual, expected)) { fail( String.format( "%s: expected %s (%s) but got %s (%s)", @@ -143,6 +150,33 @@ static void validateValue(Object actual, Object expected, String context) { } } + /** + * Collapses an OpenAI text content-part list ({@code [{type: input_text|output_text|text, text: + * "..."}]}) into its concatenated text. Returns the input unchanged if it is not such a list. + */ + @SuppressWarnings("unchecked") + private static Object collapseTextContentParts(Object actual) { + if (!(actual instanceof List parts) || parts.isEmpty()) { + return actual; + } + StringBuilder text = new StringBuilder(); + for (Object part : parts) { + if (!(part instanceof Map map)) { + return actual; + } + Object type = map.get("type"); + Object partText = map.get("text"); + if (!(partText instanceof String) + || !("input_text".equals(type) + || "output_text".equals(type) + || "text".equals(type))) { + return actual; + } + text.append((String) partText); + } + return text.toString(); + } + private static void assertMatcher(Object actual, SpecMatcher matcher, String context) { if (matcher instanceof SpecMatcher.FnMatcher) { assertFnMatcher(actual, (SpecMatcher.FnMatcher) matcher, context); diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java index 9768db00..1d56660d 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecClientRegistry.java @@ -3,6 +3,7 @@ import dev.braintrust.sdkspecimpl.clients.AnthropicSpecClient; import dev.braintrust.sdkspecimpl.clients.BedrockSpecClient; import dev.braintrust.sdkspecimpl.clients.GoogleSpecClient; +import dev.braintrust.sdkspecimpl.clients.LangChainOpenAiResponsesSpecClient; import dev.braintrust.sdkspecimpl.clients.LangChainOpenAiSpecClient; import dev.braintrust.sdkspecimpl.clients.OpenAiSpecClient; import dev.braintrust.sdkspecimpl.clients.SpringAi1AnthropicSpecClient; @@ -53,6 +54,7 @@ public final class SpecClientRegistry { Stream.of( (SpecClient) new OpenAiSpecClient(), new LangChainOpenAiSpecClient(), + new LangChainOpenAiResponsesSpecClient(), new SpringAi1OpenAiSpecClient(), new AnthropicSpecClient(), new SpringAi1AnthropicSpecClient(), diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChainOpenAiResponsesSpecClient.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChainOpenAiResponsesSpecClient.java new file mode 100644 index 00000000..37c59bb4 --- /dev/null +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChainOpenAiResponsesSpecClient.java @@ -0,0 +1,120 @@ +package dev.braintrust.sdkspecimpl.clients; + +import dev.braintrust.instrumentation.langchain.v1_14_0.BraintrustLangchain; +import dev.braintrust.sdkspecimpl.LlmSpanSpec; +import dev.braintrust.sdkspecimpl.SpecClient; +import dev.braintrust.sdkspecimpl.SpecClientContext; +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.UserMessage; +import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.response.ChatResponse; +import dev.langchain4j.model.openai.OpenAiResponsesChatModel; +import dev.langchain4j.model.openai.OpenAiResponsesChatRequestParameters; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * LangChain4j OpenAI client driving the Responses API (/v1/responses) via {@link + * OpenAiResponsesChatModel} (langchain4j >= 1.14.0). Multi-turn history is threaded by keeping a + * running {@link ChatMessage} list and appending each turn's {@link AiMessage} — langchain4j + * re-serializes the prior assistant turn's reasoning + text back into the request {@code input} + * array, matching what the reasoning spec asserts. + */ +public final class LangChainOpenAiResponsesSpecClient implements SpecClient { + + @Override + public String id() { + return "langchain-openai-responses"; + } + + @Override + public String provider() { + return "openai"; + } + + @Override + public boolean supports(LlmSpanSpec spec) { + return "/v1/responses".equals(spec.endpoint()); + } + + @Override + public void executeSpec(LlmSpanSpec spec, SpecClientContext ctx) throws Exception { + // The builder requires a modelName even though each request overrides it via parameters; + // seed it from the first request. + String defaultModel = + spec.requests().isEmpty() + ? "o4-mini" + : String.valueOf(spec.requests().get(0).get("model")); + OpenAiResponsesChatModel model = + BraintrustLangchain.wrap( + ctx.otel(), + OpenAiResponsesChatModel.builder() + .baseUrl(ctx.openAiBaseUrl()) + .apiKey(ctx.openAiApiKey()) + .modelName(defaultModel) + .build()); + + // Running conversation accumulated across turns. Prior assistant turns (with their + // reasoning) live here as AiMessages and get re-serialized into each request's input. + List conversation = new ArrayList<>(); + for (Map request : spec.requests()) { + appendInputMessages(conversation, request.get("input")); + + ChatRequest chatRequest = + ChatRequest.builder() + .messages(conversation) + .parameters(buildParameters(request)) + .build(); + ChatResponse response = model.chat(chatRequest); + conversation.add(response.aiMessage()); + } + } + + /** Translates the spec request's reasoning options into responses-API request parameters. */ + private static OpenAiResponsesChatRequestParameters buildParameters( + Map request) { + var params = OpenAiResponsesChatRequestParameters.builder(); + params.modelName((String) request.get("model")); + + if (request.get("reasoning") instanceof Map reasoning) { + if (reasoning.get("effort") instanceof String effort) { + params.reasoningEffort(effort); + } + if (reasoning.get("summary") instanceof String summary) { + params.reasoningSummary(summary); + } + } + + // Stateless multi-turn: don't persist responses server-side and ask for encrypted + // reasoning content so prior reasoning items can be replayed in the next turn's input. + params.store(false); + params.include(List.of("reasoning.encrypted_content")); + return params.build(); + } + + /** Appends this turn's role-tagged input items (user/system/assistant) to the conversation. */ + @SuppressWarnings("unchecked") + private static void appendInputMessages(List conversation, Object input) { + if (!(input instanceof List items)) { + return; + } + for (Object item : items) { + if (!(item instanceof Map map)) { + continue; + } + Object role = map.get("role"); + Object content = map.get("content"); + String text = content == null ? "" : content.toString(); + if ("user".equals(role)) { + conversation.add(UserMessage.from(text)); + } else if ("system".equals(role)) { + conversation.add(SystemMessage.from(text)); + } else if ("assistant".equals(role)) { + conversation.add(AiMessage.from(text)); + } + } + } +} diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChainOpenAiSpecClient.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChainOpenAiSpecClient.java index cf270346..50d5311a 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChainOpenAiSpecClient.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChainOpenAiSpecClient.java @@ -1,7 +1,7 @@ package dev.braintrust.sdkspecimpl.clients; import com.fasterxml.jackson.databind.ObjectMapper; -import dev.braintrust.instrumentation.langchain.BraintrustLangchain; +import dev.braintrust.instrumentation.langchain.v1_14_0.BraintrustLangchain; import dev.braintrust.sdkspecimpl.LlmSpanSpec; import dev.braintrust.sdkspecimpl.SpecClient; import dev.braintrust.sdkspecimpl.SpecClientContext; @@ -28,9 +28,8 @@ public String provider() { @Override public boolean supports(LlmSpanSpec spec) { - // Chat completions only: langchain4j-open-ai 1.9.x has no OpenAI Responses API - // (/v1/responses); its internal OpenAiClient exposes only chat/completion/embedding/ - // moderation/image. Responses specs are covered by the raw OpenAiSpecClient. + // Chat completions only. The OpenAI Responses API (/v1/responses) is exercised through + // langchain4j's OpenAiResponsesChatModel by LangChainOpenAiResponsesSpecClient. return "/v1/chat/completions".equals(spec.endpoint()); } diff --git a/settings.gradle b/settings.gradle index 7fadeb0b..f04daf27 100644 --- a/settings.gradle +++ b/settings.gradle @@ -31,6 +31,7 @@ include 'braintrust-sdk:instrumentation:openai_2_15_0' include 'braintrust-sdk:instrumentation:anthropic_2_2_0' include 'braintrust-sdk:instrumentation:genai_1_18_0' include 'braintrust-sdk:instrumentation:langchain_1_8_0' +include 'braintrust-sdk:instrumentation:langchain_1_14_0' include 'braintrust-sdk:instrumentation:springai_1_0_0' include 'braintrust-sdk:instrumentation:springai_2_0_0' include 'braintrust-sdk:instrumentation:aws_bedrock_2_30_0' diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-863a18378a48.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-863a18378a48.json new file mode 100644 index 00000000..6dbec440 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-863a18378a48.json @@ -0,0 +1,113 @@ +{ + "id": "resp_0353a669c9551cba016a84048f875487d084d9057d7cf85be3", + "object": "response", + "created_at": 1787036815, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1787036826, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "o4-mini-2025-04-16", + "moderation": null, + "output": [ + { + "id": "rs_0353a669c9551cba016a84048fd36c87d0baabc6dc9f6db3fa", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABqhASa-I4VIqhY6KSx4kYjkcZXA73TG-7gcpuRhK_jeMdohLuEsGirvpCrRf8GNycUeIpo4BC3d4LcuYGtPEn0W6RlK44EkxuSQCdVLCbSxFyf5lXEpOGS-DOvK7Lya5d_1wxnohQVGebu8G_Y8PmwoJS2VBLmmOe7PIMJ3D1VMyWfiwIOwnVCr0z1jOBpvAGBRPjn2_4p4qN1PotDmkKHjlVAVWIExZOXH0QViNuH5q0wpqWgpBVwFPAtF34hEpeD7XKato-CT7TYV9aUH9m9qYtR9vwqkzg6gm6z43ZYtZn5ZNoTEJ2ECaO_NfvKmn_Pmq3ttetUD8_jiD1kvdu8B9fgOLDtOTNYuEePht8rjS_ddGEKysgp2UR_Eul5eB5jc87d6gtgcxz1226d3ytwtLGzDQWgq3i7w9ir4JE_UBJTwF0ReWjscNEkCVHeog6n39t4R1TpRMbSUOcnkjNZUkJSUSsrD8D7CFzyXgzaO5_Tl7Vzg6I9zQrKF-5Y9ILxm3iwQ74v4X5kgha9jCUL2JZkUXb26c44FxJvVDuvcRGSLNjIR23kEiM_p8sWu4qZjVe0GmPhILJg-CkOmUZDbQnjRSPKYqzemz7mPDTy0wlGf-sMciwedknWxD6HKepY5kE8waLyM5Q3IkOIi30RRAVLGzIPc5fg14NmzF-KFhpK6yEfhgsWq4LQ0-ipb-lFu0WTtsp0KoXqO-wqgk-cPxEtf33wMlmWklNJ0FeDQxrj5M-JKTurZ2tqCwYpfrnvL9DpdM958NSt4SId4k1ybVEqCTM-bc_JtdaxS3Wr34DAw3X5WS-b480HQUGCFG69QF3mku2nN6HlN4TTqcexUfDFH7xOLiZC_uYVh6zWuEypo6OC6rb7YxWOJqJJYLvvBcq6bJuypMZfxMdlRkfqG5mM0a_Jeeka7NdSlZA60sATxQxWiMikaBw5H12_hWSMzqTzgsaesjroQVpHEsfKEyK1UlyXcH4FiQWR8eZusGoq5vtLqb5bQkNoJ0xRkDYop2H_UefS-07xvdqwNPDocpyxwGaseXwLK3PnYdC4OlyEjFI21HHvDU5ET5od2PWqyOd0jmLvmx6r4Q5OlBtvPse8C1fIjq7oPbCcgcxwzAJjVlY3PBnbxDuapP2BPFV0PkLcGCMDE3UcLnQZ7y9cTlIO41KlNtAFEBnl7U7NzDpYKJY0UslVVeWRU3ulUBoQF3VYB9e1MEY0-HO11hf7P5q9QNATH3JieSXGvGNvrr_sl8nwM1rzPomU1P-m1wxGxtDZ5RADzU0WxFsUZCBY08bYaQtIIl9Sue7PzKRPWUTMxLG3dBSlo-sZi_10j7iO4JbDQXSTZvPa26xopweRMyMNHnsB-REu3xRXHSEjv-fPnJGFAt8gURAxW2nv3LEWNKR7WX1phCrkQF6IH398MBtqMik6oSSQuYHs_gg8BfxFGZp_RMKonTIBAdlNP6dMJURFfYIeissaXNrozorRSOdAzKOxFMcI8De7L2eluBXvIm0ZArtm8zM21XuwnTPvHC0u2FnAGJVQ_vdDjfnJwGEX-TU3tv25qTwvAxfDKwkdEacuK5E_Y25uTdP1hLd-LsLqIxWxzx1B2utozU5g1SLb5LmXiCKLrm4yI7qNZC-TL-CNDD-c1PS2jVWLQhaU0AOu3g0PSnBnYOEEou_o82oK3Bfgriq9ThKbD0BZGPOWEcWSUWNe2P_dwDAgZprbJmkYVlBtYzpSgwC7m9oND1IOFBVgRD9xlnpbv21ZHU58CXnPVfOAF63wv3bH5v6kU2GaL3DbBQpGIbigAV8lRMkNySsT20zVU_5ezh03xw6ZOmnsja3wAKCzCoLKitt5PogoxsIBOVLT4r7_rvax4JZ7EShaQ_SBiWWxNQ6LXtYKsGrDVC3-7ZTao4PANZSdvHgZJfo3tpbxgIJx0MQ1Y-l-Dptt5_NdANECJ2Ro_buPei1o0jAweU-Vtx9ofT5N246wrJFY9Ejf1b8CiRGxtRWZ6Qs7qgwrnt2LbRXAyNYuZgItuX1wj-nKupCe_BB8xUS0ftDGuoUl6OKc6NSCOHpE9zVStB-wsnij3j9IoETfAjlpHYOza0zC24_NqQyALHqINrGSsyVcXdxtwXBVcOrn4twOMM5h6JqtN0dcRzZlNrvmcHHOgbl223H-5bwbezXyTFjUvLSgiLGpvemp0-vSyZYK_hnXHdFGXZFGIe8nqiLgp64RQ9qEx12CcOqSwnhmB-QUIy-JeUth_Dxq2bS_zx74IGd-sIEhtoAQx555Sg9Cf0edCXlK-olyGZB0FlG2ousWvK-OinqiltAn-ZCupGSuWSwt70wYSIhW25Xxy3N2Rja0pOsVYeZi7kpRH8doZW3cG_ShlxqIHz9g6zSgKtXHtFzoRzkrIb3bXRgT9FGDjP15WIzna0u_uj-F8P8Msi3a9J-Wdsm_xlpFYPd61xDoLYspdpiNbE0OW5krAp0M_zkr0ozxpMdNPnAJ8WHPNdzZJJagP0qaUCQyu15x-s4gunrJdfVt_Y8MB4U7qdh7u0tynx7dvL1Wx3mYfqyY-ydVTBn5PDrrZ7TRb8AGDqBtDALfaLanlPyAD0vephy5nKJFHlL8VZgLp5Yd3NuX7GN5TuwZGRwIhpmxLs-Hrdf5hllqXHyoG29VlqhdspJhzPtqxHJJ0AIG02JmGrrLd3azVYNzPes4JL7smyWFNA_7dRAgDrg0HiEPZm7Xc_ZvMe8ryCuN9ijQ3FG3WruF8_499EK_Be4460yomt73m00ZFV8So4UHw5CJMIF5ju2K5SGrqQEhLnTwhsp6fWTqqhMAO_p3JfWpx2KnBzF7eZfB6_BM_jbYzfZ8SmEyInbZ3p9-BPWJiGAd551nehYxGtQreLSu93rxgqmOvTaV-sUAX_50biJUJ9dh1DA-qms5E4ktB7GvXjktJRJrD1WaNMMa7fCMpOLTtPCa9WoAE0I1OLKVS7znw0V3-bseu7WOB3XWouJZWxfX2Rx9vqqX0iF3SVoz3Ot1FebYu45kOx9DszYQwLYeIFYDTabZDLqTKjJ5_dvt2YbPmLmkHItTpdBSOtMwKnPKVMdZUZ4V1CV_w8mTMU0ElJom9sIjPY_-A38MdtZzJFaz0zmEEBeqFQJkckllKKg7uZE0OnVUeCqRpUMrUTtRKaYFns_mM33pjBQDBePhJ7DjCGrlxl4B1_lR1W1PeALgKpErd-EKCnXltXSbTKrqwC6RiCoGsJOVdrBPTL7nAMTFYEOFpn4JarmzZsq2-0Nz866PIWQpcICpe6Du9oumr-Wg9ARMp0f10HnPAZm3GcynE9bM3bs2rvrmGbRAjxR9ToqzGdmXDp1rrRsIkyEFmFgRj5Bc_9lULYhC8Sf84GUObnVsAHhQ1DBCN8kOPJAd8aP7UAjqLAhlC8hsuhxNF3w43GV04Diz3CoCf3h3kpSZ_LAykxSYXZYb9NOlY9bgLnFpJBJqIqlpxbte2APqdDTxazz10ZXN0zZT8HohJRhLXqqWntrnFp7_em_9FgWGcOgYHkYtu5R9z7v1Rcm6d80Sl34Wn2joPioBGUQyw0ksQgS-R_JYZwDZ5Nwrq7CY6lk-amfnwwm0JkllVDVcXeelozumxQl4zCtLZ6taAwPLskmIoyRH02Ko4rrG9ORr6bkC9DfVq3jxqVAuLSkwjpTKxB7N0KiEkOq2gGoHvAmu9On5Y2o-SQbqy22ngXYUV38hHpElhn6ZS8fjo3Q72jq8HaS1pZG5jdlwGPHaafiBHKmtGMAcQk0-4aUhvcNvJ-Eyvn2VJ8Gf6nUS_5Vxydtf8KwEzGkPtXg_7ktWzHxND-ALYr2W-GJDyj4y4JYgIr4ikwUjdmFB4UmX-fnw6Nm_nZFMOTWTD72sjBoXzoYFRrp5RTY1uXLUZvJt9FHYSMBW2duAnGA5PO9VOKGIcWhF-Srq8SfHFXDhlky1ADh3I5vvD7sX-R4jRChNqv2olL-5G72Kv9Mjt_-5t6Ax9FdSkYifz_C6if-t19XyvuzFD5a6MzXi37SX7SgAYn65FBbaB-smtHLCSZkoFlh_u2PNZ_6yVThzGo-YSfSe0wN4Dxpfw8KhREcnMjN8HjvKT4Ce-viNxPC77VDYHPBxulFxmtEWn7pZVoBuDOXqzG4hOWvdQCHVyGBSiKVX6DUJiOdCnY-anKgYxQldy8-tULNMSLYNvTjsjxV0RctMjoWdkP_WHGZsLTtYq8lqswa3WN860s3tJzREucgH62FlNmFPqyXon-cbjk44_lbGlO7de5lZbW50PUqzvGjkcSQ20yOfsUInoexh70JOxLCMUw6L8fDqabSmOJ45LoOi5rGKRHN9avZ_kLdVEm13_ngX2KqEBGmocQa6rFLv6nQIrS82w3MaG7QYSR3WAYyYNKUahbywMDdWnzx7NhcG9u0q4V1v6mpm0x7gW6rjgi-7Dv6KjiSI-V1MYCU8F8DefIi7kekLNuB4l9BLrd7xwu2hKx23UGu38eZ-B6eNKiactA_G5t53n8w8y18ILsKGAAXlMNv7FHVfol57fWQL9HzPvXkC4TfxJ7aV3WNfWuabKFUaBfdPrG-bWEdfNNcnO3hWgeTSqy4Tns2UPuXTk7d5wWqBl07T6D_QqPWs9kDDBVDLZmjXDLi5sVg8PN_s0uP4C3y_lTb5VJao7LjpPj9wXRKg_Vb5TNCo4t5Apjgh3vT4qJw0RtI6mVMuCgAeYiMdq2XjasUdUY9CH9U6a1M4IGn0pRF8Xn822B4YHwxsCMl3-4i5RrbtGO9GId7mT0YELi77VJg7MKA5yiAI0oy8n7xcjh0tZKa9n_a5h3vH3rUOZ2x3gvPy-JG_3J6UENs7oSAfASxd2W7IymdToBP2zltKn-lq6gas-pVCFy64ghZvyI5HTlFlxZc7QP5hLPBFmJTKTbYWBxRi7IAWn5JY08QWFpSgCnZG-PDEd838-RxSWIUj3wyJ70nnvjoz4AVPL8gTwu9RZ3ZyPDD4yxVV1bpWw5990iRtQztsDpeEiQ2Eea1Q1JVHcMjJx3gGn7yWUOwFa8InevH9sAia9D_HYQY3oIuXe3DX-pdXCxMdWvLv_JqpuU-5Mu9UjXxHmABAgvjHkykWwBsYVGnGZcGWAlpaoLkkugk-Wk5Z5jsIppqsH5KXbVIk_MZIG_glr-f9bK_3SSY_bGbglV1KIYArq1ombToAe-m7BtYx_YJpXCAMRfltSclZk6WyN95B8SWj_d0UnrUKXDUPwtK3WiIJB3ao8rSfEpn4_46-gYnAnOwOnC5Ss-8t1PQ4gc3cimk9jmIhbK95Su33Z6WevZlzsyLalL3PKQR-_pBs0txL3oVSxxueLdp-mCDtdkKYyZB7Ql5HGfqvvEfXGuBjlaJHH0oKAaf1j0pnx-PK399ggDEkyNo-Gz2cqv_0EeauZBOivQugCKq6P0ceEf09_eTajzb2xfcpEtuqe9vssNxEeIr4jMucTf0_heRTbcXjlKEPjsmDk8PpqvDF5REdPVOW0c8f6mUTN_6d5D2UTAH0K886KjMVFLJlyLR7LMtNwGfpgjlWxlF22riTmGr2lKRGBj_gX1-kttex0kHxOJtC3-jEZPLa7svAU1yNT9U1fNHJbI9jsJ2JeF_8CDID9uM_MhJbXmWAa9AHBGWcpXtPCScAJ55vOcXmLbKgmoBy46ROMMH03S0HqdOLwwTMMdX5KBCs98GpbXAtz-Acr1ueLMuOXiNWTjKvA1dqN7Cqxus5gJQEvuI_yS6oeiVlGWn0Y4z48kbthXl5Icw48XpogoU3x_S4-qEFf6iYBhHImz9dCfsQ0pWNBGF3EFsOISD8xgCBN8-ZThpeew-qF5YhW09A0mVRpseOJZtnbbDOmvED11TI--sSpyatqVk-bov5VG4EqFzr6VZRpr4lemodUL1uZXyP7HLH4ZXeCvPTDE3qjHcuH4HvSH-m0WPvzNLlmEPOODt1Tnw-3p8LXGVYJbU97-zLpmahT2c8CIxZeeg0OOgxCJgL_LtzALfOXFEmawVWzm9BDJwaOTytMBCXpDd8S8y5p0mHJkiaVp0yfajjUu4z7ZBRV3Fa1Z6exsU2HoUztUo_o9T7MSTn-tT0Bz3HLM5DFWj6oB5zOvHYplqHSIWsOmGmezN8iI_5B9Lg_8CjJ2WkbsimLcuPnCRDJf4cgKb6B6ftA-TIPOJsBq_9Ahc3qW6WWTO5BNR_BfMDxEwNTagNWjA7V6WGaaO56nzwmLc9GdTazx_7JAiTaUqenguBYKzfS6dvuXP68DJeq6XAUBPefo8MZMtqPl5CjwSNrQjraqckvolWaRzRZ0ClfoAXPWwAMztJo1jZv1P-KN_2LqAx52MKSQ3tVv03d_2u_tZuZeDOMd1rN08zvgWkgubJmwV_91Cl0AKwnTlOMySaQ0thwUCeT-bz5oGwJOdXoVr8247Nm0xNxtzvId_QhXZ0snuPnHDSNw6Paz9x9FxoQ4nhS09ep38uANyuwFkaN2dUnVtLvt5m_-8pw5iHeCo7hpl9P_DDsUTBsRHG9OO_QirFCSife6XeEvP22yi3IwZj-CRJ07YtXEM7ygdsu-Sj-ATzQFeSulUixESO407jLTlpPT0GGqQWSdVaSC3969ORZsEMrOoVAbEnhmpr4c76n0fzzlFSLETMHU_7HdfrcNKfqqLtosBOzYPonieFNNWGPrOsj88m1ei-ENcuSsaX_8k1tkZbnAFnSEt0gqAlkmpj7PXILVnaGdgemi3iLDd1L4PLbtHkqZLtzNwyAuUoiBE8EaUAijF25A71XusjB44o0bVfg8ObBbvTkKf9_NgyPCrpUpYs4bEpEfisOZ1mqezThF3862xKPWQD8XKBjBgMBe9xoPq582wtbEx59fhLaZQ=", + "summary": [ + { + "type": "summary_text", + "text": "**Identifying the sequence pattern**\n\nThe user presents the sequence: 2, 6, 12, 20, 30, and I'm analyzing it for patterns. I notice the differences are 4, 6, 8, and 10, indicating a relationship to triangular numbers, where the nth term formula can be expressed as a_n = n(n+1).\n\nThis suggests that the terms represent consecutive pronic numbers, which reinforces that the formula accurately describes this sequence. Overall, the formula seems simple yet effective!" + }, + { + "type": "summary_text", + "text": "**Establishing the sequence formula**\n\nI need to clarify the sequence starting from n=1, giving terms like 2, 6, 12, 20, and 30. This aligns with the pattern of pronic numbers, expressed as a_n = n(n+1). \n\nIf I were to consider zero-based indexing, it might yield different values, but since we're starting at n=1, I\u2019ll stick with this formula. Consequently, the answer is that the sequence represents products of consecutive integers, confirming the formula as a_n = n(n+1)." + }, + { + "type": "summary_text", + "text": "**Defining the pattern and formula**\n\nThe pattern I've identified is that each term in this sequence represents the product of two consecutive integers, described by the formula a_n = n(n+1) for n starting at 1. If I were to index from zero, the formula would change slightly to a_n = (n+1)(n+2).\n\nThe differences between the terms are increasing by 2, confirming that this is a quadratic relationship. So overall, the final answer is that the sequence corresponds to pronic numbers, with the formula a_n = n(n+1)." + } + ] + }, + { + "id": "msg_0353a669c9551cba016a840499cc2087d080f07202381c973f", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "The \u201chidden rule\u201d is that you\u2019re multiplying two consecutive integers:\n\n 2 = 1\u00d72 \n 6 = 2\u00d73 \n12 = 3\u00d74 \n20 = 4\u00d75 \n30 = 5\u00d76 \n\nHence, if you call the first term n=1, the nth term is\n\n a\u2099 = n\u2009(n + 1)\n\nEquivalently,\n\n a\u2099 = n\u00b2 + n\n\n(These are sometimes called the pronic numbers.)" + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": "current_turn", + "effort": "high", + "mode": "standard", + "summary": "detailed" + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 41, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 1529, + "output_tokens_details": { + "reasoning_tokens": 1408 + }, + "total_tokens": 1570 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-892784bdb435.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-892784bdb435.json new file mode 100644 index 00000000..dec7dc82 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-892784bdb435.json @@ -0,0 +1,92 @@ +{ + "id": "resp_0c5fb79f7a37e1c0016a8405e33b8487d0bf79d1c1a577ee84", + "object": "response", + "created_at": 1787037155, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1787037156, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "output": [ + { + "id": "msg_0c5fb79f7a37e1c0016a8405e40ea087d0867b05d0b25c00ec", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "The capital of France is Paris." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": null, + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 14, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 22 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-bdae47d54959.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-bdae47d54959.json new file mode 100644 index 00000000..363c98f3 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-bdae47d54959.json @@ -0,0 +1,113 @@ +{ + "id": "resp_0a7d8cec3288e849006a840490830c8199b3c498f135c8b436", + "object": "response", + "created_at": 1787036816, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1787036827, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "o4-mini-2025-04-16", + "moderation": null, + "output": [ + { + "id": "rs_0a7d8cec3288e849006a840490dccc819996b53c37df120094", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABqhASbJorYQo9xxCWixpp9X6MBEKXknGhPaV4Y5di9qeYJ97cv4Sm8m7lgi-GxF_tADp1kIhJUnLOKGZFA_awKYpupqjU8hQPPXq2l1sK4tBNmGCPyo-HxoN2ZXLNI-8JJ3QrbQNoBZKSjF3YzdkJEoqBnKec9jcxDP9v8toTCw2JRoJ6QyqTOwc_y2_8K0qxv1qSZLOokxQ67uV6prq55I2m4LFu033ITv4ZvEqZ9DxThDpwjYTzn_tkgR5NGZeYxsVO-oNOLTD3ww8a7y3GbEHKQhTF9YU_Mh6rLQYdbkN1u1HT7g9iYzhz5uf4A7CRkdMheW7Iv3onS3HtdS0YwCIwY9jZOU0IqDSkPowalTtUWAt55_SDFT0VLMqUSfbITXuhSQ-8NwG4J81WDYgu9o-nQ_uJkodTo3WRK6cRks6cqLfdjhT9OuiXVm7y6fKUsSBk6Br8j5wB7b6zDa7xQbxLu6G2SaSzG_EVURR-GMzBD51FBpIFq6qpx9rkvFI6rNGSzkj_kX3pTJYQyDbQv_y7xOEiC5a7XVtqSFM9Ng38kNxSaaHguV3SA2rPRRKwbjDpCK5I550Qx1XB7Xsc3i8QfqofgcHcVzGmbfYLCzkNQgtN4vMj5Vbd_p0na1-eVUSsYnA0ZcPwqrDBHwus12qJXGHkblaMb98h6D0IGZ6FIq1Yyc1i8SGKUWu1sWGJSwfcXYzmadRMjJqdCa_F5IzLu0tJVjqC5afxgr0_pPoUL28hc4cmi8QqeW4M5XLwVeQ4i2zsToy1B9zeI2IK-LcGiauAGtkwn4Sqa4woWH3KC4onIBcFN9jGVu62Ati3OwTACGVO-u4x8ZGSMpli6bI_4ihn1nk60I_8v6wSuziP8eOnYhwLL7JazGGQrS8my8mhpDdjxPFl0SK_6W-stnnCpkgg3tq4lu0vlsGPETo8rGXbRoESsw-Y8CiGtqCLki4ZIAFqJ7HbaJcWPFTKgOcoGTq4Z2dOKZD3f4tQzitglg4EygzSjBW5YPfROS6h6h83Xv56xfL7cfO0KOKlnNoZxsdXVg1QCMTyxNTXp9RhJHqdH6VCC462fR2ETTWE2xoszjeP72evQwE3z5NIhUPmx9UOSFMajmZhE_p_nAPunGD2qiUyU3yBmPNh7Ko3wj_n89v4P4VnpB1sPA4JXitcBNDdRu4VjwZTkSfmhbD5yspvy5I64tqQ2BlJDfn3-B1JQDUur1ihDwNj7bGWwa42mKfL_GHr4OrFK8J8l-s3hR75HLya3cGK9el2fEO-bgRWCbGI3HMMiZyAQHzA4XU7LK5HiUe-iNZumT2hDP52-XCffIRnivNL6wJVsgd2EZmntk-TRuaqnL__TEkzEtcncNactyX7KAHhfMpZIdUY7tMKRWNMQWfMNBPlHFHjWzPQXjvTAgKdsCc8ob4eljfS6VF9OeaDDe_BnNSgFk6gCyWZm8q9plJrnxhYd5Jd62xxpdS2fq0wO9lhmljitTCnILCCeOH6L5gD-1xh4JlfKcqAvTNtsBG759O9l2v1l0z8QeWw5neBhXTjH5PDS7cpqqmm6r0nIT3BwfBxh8xgZGFMPo1ErN1fmOCQ2tmIB9YGNYF32V-RoluaTwL9Z2CZMPB5x4ndaM_JK26_UbU80wsrptQcmpikbP2A1DVVCk0E0YykLmTUJknGN5VDjpSd_PYAtOHw4a2nXzdEP_quE6HcE64jQRznRChVNPUDOTW7Uhr8_V_gWoC1sVykrYmMsFSE1h6_1Bzd9o-OJg8z3IfZLr24MDcBBnR2MOPVuPz1zZINdsRICTOHNaLPg7AakOO3IPSrKJMLkJD4orF8mzaNnUIx2QuZMuKIeHWdL4e-eUsyfAxKvo2q3WYjOeFADz8elkgDKMjJkwzop8LYk33C9yrxGYh58wlxyLIoAiRQODk6y8YfNk6QNdgeclESjUVFKapGtMvOida8QHe34NSH22nN5oMNDjigYIGqLAH83ahz3zl9UNwW9Z1UDyDkqVuqao7JGQFYVlgo9fQLZEpXH4Eb_m1qFpAoyGceRDgVpg3JuFM_fmtx3xwasx0YmdVx14_aVqS4qhg0IX2q6RQkfRw_2bSaiJZWtjnKUCePCNFu-HaRap1GaUbhzom_aYSxIQvELPdhEiOrz7BffoePstMeqD_Mtq7Ow03XnC8be6Oiqrt43ndq9k9W8QaKQJRDtwZ-N-PvmDoqbfF9Zi984bsAcU-mA-n6GsX7UxJ2Rfo4UvOHHvEZUClD9XhMv5ov4VPZATguOGvEYe9EQT3MixC5rIaNGV1LyO61G_q-LdmRo2uaEF3dV5UXttQkyWpYRQ26sgB_qoZ8_x-MS25GD4KvQkBHDycoQGkidwgR65DXTlT8241wB1fu-tHFCxPqnbxRqgsOttqvAUJjAffBN-6lmLBwBLzcllcr0MpSvKYP5aqXgrSPBKWJvcV820F0sc6NSVNn7Wfs7euiCxfLtrt4-Abe8ZaodKD6oaqOYDHmdK5RqwtoU0_dFhWf3_z7dumFjhBo5OoeRHCMDmw5kDIYRuwwqU133gsnxG2jVHObiUMFZLDYAZ_2PYYGsovAjCKSSFeIcg7Rkw9LYiHUHmsK1AgfORvjz354qDTugoKfRgyFpZKG0NWmNcv9VpUHI5DL_sbQK5qJ7ru5InyORURvK3xwg-JRgO3UDKc_B_8TtQMEpH9VD09ihm8xSD-zgG6_QLCds4U3oZilA5Mp5G_szAiEIN-E-GW28SpixkfYe47X-7xdHa-Rga1soXFLR8VxuyyK1V3dvtD13q6Fgx9NgOog79YbO_XuFzQ78GoqChMmEgmAubM78IRKy-GfMi4xf06HS9Coh6NOBpazNwrPR3S8I9vBoIZ_eJwy1TQIVn1xJz9kYW5aMRCgb2o0eqwVQpm7xlPvPZFjixqvLLhwPTbfyyMgwtY_o2VW8oJ_9onFRxh6TFLpVchAV6mOtBOyBDHgxpjupaOhlFKoplx7TR-9Pk4o31VdnHERhurRlkum14snPjYiX8qlxyLdMZkTLAEZkOVjee4O2oqNmdzOi4ISWRyhXLbuaLPaWUl0USMld9Xx9kv15ehaSEunTIuol0jQtpwt3Ou9HTqqnxqXIkyv5iAcfK0kDkikZU5SYZHcia4GQcNq52_dk41nRzAAbBQIUI2jyMKPmz8seqMlPJvJ06B0ZLXI76X-kRpe6Kwt2Bj_dlDnCdRUT5VcXBGegzx6QSCr00fGhQ46rWOOuStLtxenIF6rLxHOH4bqpaoJLQEF2IVjRvxCauBTsXsksfmwTBjBavqtAf27LvyV7cfo_m5497r5COTR-WG_vTJ26SRaHJrWS73V3HCnc3dMYKKvou73_wyMmI4tcTrUkAjMXbGilHdaMlSi68v7ldbghwz92N67nXO1s-3umzEW8c5dAiMyrSxQlNCdq4d_HSm538V3yggH_BqkFTz7RgGUlz4XqKzETfI8OUJG5RWOSJXQTGkTTn26RgvIdBXRTBizLAjHi06GpCnXKQDq3umXFhMd5iEf9_Hlq0cOBtd1EtfTMCN0S2ecEX_RxRzD3f3R6_zhuyHLQdgIWWGwlvB0ihn0hqttGPTxN5tUYLujPxWifpYXjD_arcWYC9NdPQmr7eAyQ9vRFba5wpmFFNYHKqOcOa7DR7OfyrjSWJXflxbB0OQ5Aso4z4T4Yj-YzbmRIxx_5-2ok1-YZjs2cDQJvQij7DAKV5LzztFPcHG-lp05RsOX12C80_6wVZ-jTSGZHrjcXK0QWg030sEusqZkBJRv1aizoqWeVtellu9dXt1A6TUnH0RsKiLbD1BMnRwLKperCXaYDwntBtu9LYWWQSBEAl4T9X4xxrJWlW6PyZzobwvS4qLpDPxq6LUIDWaBuJKFqC5i0hdv77X4QQsaEaYMSsfoVlNojIBUruh5uKmLEcmaXKIS6MLpipKr4dml5Pn4o1OTrsWERv5ReD09YVlYYzrSo1PYmBLzlUrF7j8vbPkusVF1RKyrRK4C6CZ5yIYSJZxWVXN0krEbyjiQo1qsfRfJmHACe5BMZzd4kFYEVxVyFegAmMzK__RpvLAj_g6HOM85q3CBymlJ_kEx6GD7fSfKTdw_B308-UEnIywLpa0CfYkfiFNQcCxhf1Jl53Tn8K8r3KNZOPi57Qm9G2uWDPZ_iATErpdLHcfACkNiQlxL1Q64deTaQD3K38hP4opghp1CnyxvVwb4FEuGi7QZYTXbTOrR-c5KLJAqmsxEuTbySFQfk1zMOIB0LRX4KJ6uOc4ygr_p_VttMXE6jZVj1dAbXH0eiySOvi61iG2JzBh8zFO3blP0g7tPw137yCRPqvvnmd2Lqj_JZTxMCgeO3ZjJhPPsREdviCzJXJ-MQuHOLEKBz128Xtr-Li7Hu6yySOXZfdZsC2pLs_UwFD_QUjoI8dQu1ufq5EjnhbjBIfOtn6jFoI5Tv4QEnGb6q2RyjJtmoHtjW9FBq-3zCdf-uZK8HgehoCw-JMvLnqZvmofBfHwpnsx-4glY33_JZMpOBlLdRlDb1mPAuSk92UjZM5PxgcStSBknbdqNy-OG2_XbfwN1ARKhpYoBwjdk2GIM1nm_ynn2rG8h_Pha1zlsU_KD7C5xenT1J3u6QN5k8y4KQs-8NIwfM4XaBDAfjMgWXoTGE7Hijp-KTmUFv0srOYUbLAC7_C5AWbRA1dIsd8O09QX7z5F_KVgQrHeQhvK_qQKkLbmjxjkAmlS82t21Ue75pNWwA5Ok1UCt2IXvQJWX-lfWprUDT73f36M8MsQT_RVkmICm0Oh0tOSbZDX5F5Cc401ilgsevof6jkyINCQpuTKzpyXiFE1ZJUz9QClJU2alGy_egz81irM9pzfjHYBGlVfhKbs132dqJeiMyT-cK6DrUZQ4Zl0PEka1Ngip7J8cAuFtIAE5Ii-e3PssAkxoQjJbpG-h-zWhuw7FgQPLXVrluOker2CBRYQoFsoyQm5r9ALwGWQnJ51Wg7te_tdZ4F2U79lP525QdmYqSmjstdJ--jEnmHwfziZO30uUoRH35eLMXdPpME5qA26LoVKJCQOflRUqlx5aXKzbyBKEOQPudoW6OE_DPorWuGdIZ2R4P0B7sS5pkaMexAWYGyGO0BaIbvn6uO4ljs0PqkjoCz4RkILTMTbCKUlKAhX05D3yaxNO7P_DIKTwsQrPXF_EiVyKs67IFlj7TLKW70JX5EyZqmJxH1PKqZM02bJwT8Rq2ktkTzk6TmciG6bCk7xLeDGqlgBRyIcmZtEYJdKaEqe_VzsYtHxjQAqejN9r5LVI4kHbdqe4NvWfZC662q--Iko8JvrEkhmZD7pT3YdGiKfiax_ckYmPSEq0uBDv_Ph9TLGp3RxnLgrY2KmTMLptP7ZBlXMPUkOSdheNH3YF09kEC0sEpyhM0zL_HVRuDT1l8ZjWhzapAjOWQwZJM2XpYOtGqr7uRQCipUtkB22lqRNSEYHkSwS50iY8shv09dFxkqOjqpUfSF_txDjbTJIJFwKTuLsijnmN3QW677B5KjgSSMNEPbWJJ02Qpb6QV4hUu3U8ORL6XhN9GFsSVfiYDAUBEMCkJSaZ-aBi3Bq41WGjYfr-LvR9QiAL95PHUamHyqi6R3S2fMijfuo5Fgu8J877UHqCODaehReBTBdLMuupqPq6Dz84RZnMLLk2mhKpup8iQBUz69TwMzf9UiU5INo40XwEhAE-5sLSZ7uVp2QDUWchXemzQrR9w_-MexJR-wn6OH1jdesgrM8L719lcLgdttTM6XT8yfX-R5geIxRt0z8sHiVGJVTvUPnZqOdX0J_lNIgmWT4Kxa9ysnNT2RK1ajrDL-nGII2ansJUth4dFkIubQ54V42FjWoBrY4nOKjgYQt0ZNrL9HGHM6YTM9ne1PJjBVn2fYcJytiAy3lP2AIHW6VnMDrLNTTIoqZOyxYBhI0XMaMzEGtNczRzZNe-u-STLiUkUWSDlgQXNdcHRYLYNpAeMkUSSDzRLBkH810iUER9TBRrnTk2Vu4GwY16GrhKYTlRhr0u6NxYZOoC7BQ2qN0BeUcIOo6hxm0A6SJMkTj8t6tqOkTePKRN-QRm6G0YaOQm4w7QB2F9SV7DAQAHN3nzHXfwW5IfVCVKmetaNg9-fUNWdZTNDESO0SG_uKK2z8kntsqiR9p_j1uN9Ke8JLBGUZDfyNJUH4F6BHbBjRH5Yp9NjJhIZZTljqbr6Ms-fXyqA0k7I48VyHMeysRvHwbgzT5PDo4Typ4ARATb3yoMdm-YCG4ff2g4rTQA5NBIlMJiv11O-1-8ehEAS3HVeFBZBLVsprHMoAXkMYZe6fKm9QMlKM_P3L-0yKuUqDMhCm3Xtfozp4aTzQy3N28XxXeFCLMTYHdKQKnI5LwzhEjmXwJcP1P_j2YaDaVry0OJHZ8BXYxtLCptGKyFsN2txCv71GdHjIQsz5EoLm8IhrJL40KK3fWDIfwLfzYjMxEobk3wsNmoBcyydCTlKQ05mvkSUVZm4MKj62uf_F8O-2ukc2xfVVZYZ2cr6bloimFRuBlVKj2D5YKvvKA5s6FEb2-OKjUxYGLbcZ5p5fHLorjj1mbSFZt5srooIwZiOhOQFLOneySZSxpBpREYx8Qk1UvYCErGBjowRVoinVGruLkNmc1ss4SF6ATcJNW1HAdvCq9L_rAzvPXabDPyGEtekj77USHYFWlixJ5Qgl4UgsJGCmEmdJBVF7qlKUE0VJpyjtbMNoURPDvxmBjwZao7cGeERMQZ5UqhkJmA86ndE", + "summary": [ + { + "type": "summary_text", + "text": "**Identifying the sequence pattern**\n\nThe user shared a sequence: 2, 6, 12, 20, 30. I see it follows the pattern of n(n+1), which checks out: for n=1, 1*2=2; for n=2, 2*3=6; for n=3, 3*4=12; for n=4, 4*5=20; and for n=5, 5*6=30. So, the nth term formula is a_n = n(n+1). The differences between terms are increasing by 2, confirming this pattern. I'm confident that the answer is a_n = n(n+1), starting from n=1." + }, + { + "type": "summary_text", + "text": "**Clarifying the sequence pattern**\n\nIt seems the user is asking about the pattern in the sequence 2, 6, 12, 20, and 30. The nth term can be described as the product of consecutive integers, so a_n = n(n+1). This also relates to triangular numbers multiplied by 2. The differences between the terms (4, 6, 8, 10,...) indicate consecutive even numbers. If indexing starts at 1, then a_n = n(n+1). Overall, I can confidently say a_n = n(n+1) is the answer." + }, + { + "type": "summary_text", + "text": "**Explaining the sequence and formula**\n\nThe differences in the sequence are 4, 6, 8, 10, indicating that the terms follow the pattern of pronic numbers. This means that the nth term can be represented as a_n = n(n+1), or alternatively as n^2 + n. If we consider triangular numbers, we can also express it as double the nth triangular number. So, the formula for the nth term is a_n = n(n+1). This captures the essence of the sequence: each term is the product of two consecutive integers." + } + ] + }, + { + "id": "msg_0a7d8cec3288e849006a84049ac1fc8199aafa166b4289d0ae", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "The \u201cjumps\u201d between successive terms are \n 6\u20132=4, 12\u20136=6, 20\u201312=8, 30\u201320=10,\u2026 \ni.e. you keep adding the even numbers 4,6,8,10,\u2026 (which themselves go up by 2 each time). \n\nEquivalently, the k-th term is the product of two consecutive integers: \n 2 = 1\u00d72, \n 6 = 2\u00d73, \n12 = 3\u00d74, \n20 = 4\u00d75, \n30 = 5\u00d76, \u2026 \n\nIf we index so that a\u2081=2, a\u2082=6, \u2026, then \n a\u2099 = n\u00b7(n+1) = n\u00b2 + n." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": "current_turn", + "effort": "high", + "mode": "standard", + "summary": "detailed" + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 41, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 1527, + "output_tokens_details": { + "reasoning_tokens": 1344 + }, + "total_tokens": 1568 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-cfec8f65e9bb.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-cfec8f65e9bb.json new file mode 100644 index 00000000..badfd2ff --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-cfec8f65e9bb.json @@ -0,0 +1,100 @@ +{ + "id": "resp_0d21b0406a6719a2016a84049a7b4c8199b3d6e2eb2663dc85", + "object": "response", + "created_at": 1787036826, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1787036830, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "o4-mini-2025-04-16", + "moderation": null, + "output": [ + { + "id": "rs_0d21b0406a6719a2016a84049aea9481998b30e6e2df7f2fdb", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABqhASeeGw7I4ty-IPblKxXoCWXh_54iOF_XJNl_bson1A4Xf_64T3j5ewogy3t4mnFjHMUhWHIgIOGPXQfnslrIy36K1rKSOJQ2hSKla24CUBH4U1qFLtJTBIMDTsmvqDABcKeBzEMXWMOhGvvc68-CWIhDKz0Sfk3gk9E71h6LjlMsdOIKOhWBvyl6-EoQoFDx0E3Tcb1SF3HxVW9W6Ll3j_9O4UkUQrBw6CaCjoc18j8XRCz8qae1O_BPQCeY-fiV9EPpoUrlVcvODRSVbI1eOJ2r_ZHHQccAAi-XuaqyMzicR8Tn8QcwS_n9rXZBSwcrfoR37hk_6uRZzEdfXeFrHdjugiSaH8V88fqOL9GGymLFvweggle5Y9lcH5tCGdLLbwsAloi6M84Hv8-ifmSygVqHNTUBs-3WuiyL6uGjY_erIoU8Y8osXQRrKrAXD6sqQalV3r7c0-NjOeOUtOpJouVrqE36hj7iu8el26fDvB9EBNVb43hSVlncXe3PHknPO1sH5mxhocKAOk_Hc2wWMQdGbKJFWqCdBOwNwZAx5XZWp1YUYJwSJxkX03TsiQGalwLlVZrJvNlOS0XnBwUovD4A6t7wdIR7dxyXZxxOw5O8iV6qDrYNbOX2hMBtXNMldtSiCEH7XgmvEUl_huXmhq94CiZjxMmHFDRYZihuhQQmZZn9rTookH42iNrIpGQL7pLsYDIoVEIcY7p7PZG5iWbrAyRfZOeleUFq6xOHnv9rXPjbdR6-PUraUqoGY4PpIzEGGGYt3R7uUUSeFh4ZFVQVObi4xbp_EyJPknC0mRDpgO_LoiV4NeeIEJXumKinOeS5PC18DaqQhJwf_ExwtvRruJA3skqHpRtvJza4n7kwWFBUCopcY2G8Uk9wUMUqCQphZ-XmFTPQ4D4H3Ze6enbYv4NbqRsvhK5N3cSpxI-NUGCpQZ4-va-18gF6H8cZz2XU_0P-PNmOFbJS1LjG74uy0XUGVnTJzdokiU2ue3VrBv8EyeBui2-4XHaxM6MEskI0bvlhKkZxVGxlAHMlGgkQjrruRCLkdcjDUl-2xxgNhkUypOEVIBE9Z-rqgvuuu8STdRj88A9R2JTpEDjof3cnxu-XFJGuTwwm0OhUUnhrRHSDkj_GS7wO5yX9BDHx9PdwlWdY4ESpByHs7CtiGeTPmPlvxrZbURWBbN4O7TV2ewy0Il7_4S-3O6Y_Js95V5fCbomLWwtr11DmhEwH0vcGr8hg1p_P1uDAqEAT7IQ8cC02OGF1AYMuRU6vhfHDu01Q7P3FhhsQsCtXqTPmtoxWWDhCOkzwvbBMyPrFIE3DhqyTPxYmP6mmpzY_MIi_XG1AHb0eSaGW_uuAD0j6ss91amDaeQ-7AIAdPB-xkmeu6iZLsqfLC25r7X6pUlU9Kss1AaJYr4zDYq1wEUmVhZJkZANxQ7mEW5PE7wd0nEx06sat8kQSI032Uxeoswv9ms7xEnPa-e18eQ9m7YuxoAlAXYDfkO9qiTNRdizOYr_fXj6c4V0LOK2OSlDjrEVQXWrNa9WX7M1pktCTN1P06kqS7Nyor5Db7WQb5RZk2lJ1n-vYJ-Ka6VdxvwFqKirndyzFkubEyfAcql5EsEawC6uMmc86xogHWWbl6FfvTxPTxDlL-eVSFXXWo21WoB_uCr2e1wLjpQBO3eNxNC1r637lqrChic-fa083Uy794OAE-T69jIC3MVN4kP75MBRsP6xxBhmsepERXM-B2_3S0Five3osbzZkmL3QBdXdBJlAHJi6i6bIEtW0LYXnqNl6dACRx8l7NKnVN1myzgnyqctTuL2KlNXF7tXix8WuMk7eeOrFvCxgnwpgkFN2QUAXZLwesMGZ0_w1eWAIDmEOlFOBzRm2ETwdwQA5Iip_NCpzdd-SuuAkLgFxZtbmv-_Q1-e4IIjAmfM_FKQWpwTprA7laCvWOBLSsHYpTAAWh_GYWU8LIMNfOCP7385ubSjD0EaTUxKFacLDaN1i_2HUg0-4GmJ_k3G0nt5H0eIx_CMQ6KMmU5TylAyjo-xmxlO7SJaS8OlZesmrBr8x-MCfjedlxkDdtgwzHIQ5HVMOPsy7iDSqYWHFoaF8oCm42dAsXp0d_DZYx0AZ3-GzPjFPZpEDLSJppbxrh2oye1a9f7Il7woKlXXLlxXaEE_O-GA8il74KF42EUU_w_y7L0f0hjcrPfBOaAueiONHY2KSUtXrjo89KptIP8DOdG7gTQuKv3MJiCeN1rjghBVnymyRxLHPsx7svrqLnii3r16IOb3Vm0p-7FOnAqsLp38m02jjyiqPMKCgbMUPXAvRSvPcBJGjGW_4mEnQcYtqtP0e3TMTrpfvL17A38XKz2UmM4mUvOFjAuIXET3TisgQe88duBnocHljMbSzKDBIt4vp3EDR3YuqNXFzWchs3HjApNB8AWy8JYZU5fr2QxR_t7WIDE-HFUF2LXKmrY59FUzrOPw2N8=", + "summary": [] + }, + { + "id": "msg_0d21b0406a6719a2016a84049d7c288199ae689356d2a627ce", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "The nth term is a\u2099 = n(n + 1).\n\n1. 10th term: \n a\u2081\u2080 = 10\u00b7(10 + 1) = 10\u00b711 = 110 \n\n2. Sum of the first 10 terms: \n S\u2081\u2080 = \u2211_{n=1}^{10} n(n + 1) \n = \u2211 n\u00b2 + \u2211 n \n = [10\u00b711\u00b721/6] + [10\u00b711/2] \n = 385 + 55 \n = 440 \n\n Equivalently, there\u2019s a closed\u2010form \n S\u2099 = n(n + 1)(n + 2)/3 \n so S\u2081\u2080 = 10\u00b711\u00b712/3 = 440." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": "current_turn", + "effort": "high", + "mode": "standard", + "summary": "detailed" + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 181, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 581, + "output_tokens_details": { + "reasoning_tokens": 384 + }, + "total_tokens": 762 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-f0262cdce49b.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-f0262cdce49b.json new file mode 100644 index 00000000..b6549718 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-f0262cdce49b.json @@ -0,0 +1,109 @@ +{ + "id": "resp_0a7d8cec3288e849006a84049ba58c8199886e44fd47d081de", + "object": "response", + "created_at": 1787036827, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1787036839, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "o4-mini-2025-04-16", + "moderation": null, + "output": [ + { + "id": "rs_0a7d8cec3288e849006a84049c0b888199a810ede742f34032", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABqhASn7p2QcG6fNG1jbqkSkhO2VN86pwR1duU_s2AppKigNiEuhXQkC2e5uG57_OP2y5M5B7m-PaXhcNSXIQ-3NFJUqXUOWS-7jiXJAYjRGA80_i7plGtFKXiXNf2izpWWrZ24_2Z5nt0GtvpMvzNhHDvLjr9umrG3KMM6ZtGRsINmQm6vsvaSMb47jvKhwnwmYgu5L_ySQGCqkRiUpTsjYtZIzyGYGUR9g5s3waj1K4vDBfowa8FFWTCCFBkdnfHedg5D5wfcw9_Q3FPU-aOvgqEwcECPWPbInUzZfyjalZtZa4R8YTG1vKFtKlSU8eqdu1VI4j8CEfNOJGq65PANC4u3NQXIceNH2P1Gl4Zz2OV4e7gkh3Ej3e6ecgl45EgAtVw2wa-i5k8h0oiyK0-zz5QMOZsEU_DLqnNWSFkqkF4TyALIjDfF9UZVB5Gr2shq6-DsF3OWc8DLqLJJVepxCeTnxBaQQ4mkVNEUQ2G58FEV4alxb6PPFoiRGsIGzehV2z9Gh1R8AY4esERfWdclNXvBeF2HaQwEEsrNpDjlt_b9JUCRy_OB37ksIdK7DKgo9I-ixxJkmSZnJG4ZzQkj4ti0kXp8QeOV1wtZ5VZtFzBMEVaWe1z_9EmdfFcUtMFaMGDYi5TqRgpbjs3VwRkr3Ig79Z0rpT56dJaNXGLBUPBtgIbeoG53bxWpVgj2ZncL-qzaFt0GUIfbVaYBLtRLzsc1iAPgckj8N4oZFieb5Hk7udzd4J3ryodbZdEVleTztbcCJpNiIVSU8uOGz7o-l3qgUm4T5Uzngb2jFBllXuaDtrakrgkt5BAx-sWQpUa9_HEm1StAqDlsWVUdCU2v6azM0npk6hYTnogXJfsy4o1b82z00NvhJYJWL35Yg-dYDqq6DP34VFd-o99OSEyiLiRtvpSpcadnX891-74h-wmPxN-9JLGbocww9id6NqPJW0HAs-J8rMos-MmWdLzOi_nsZXCV8pG2t_Cd8bxJTBQS-79P_x3dHLLH4TEi2sadvsiELuTlE5TX7EcODXIPl35rdZPmU5fFIY-NwGaHlJcXlChpwvUZGg8bH_3uom3KOmnhwxC6qXMkVHl-UYqsKTgKlQmFmHVNjBiV8lLn40rIpQg-4tGxOdRcmrwWNXrArzS8XRulRIy-TJ6_4Jd0MSMc_-b3TNKRwg-6U8iCzoExgvevxSHKZZ7aRqYhywkhkUwqvrTsDZ3gXuzfqaxTPrVvjfMDxox2LDSiU_LWCYZomlUKGcLQsNMjsZjn92WzwkwzZE-az7s-R0XaybehIshn4KbWxRispavPSaLKHzAmbsJG-xZt-fcTfovabmpacdl_Q0CC5hz472ObJt3q1JJ1fITIvslpaof9go-SDaTrFRcjlunuULPxNHjEaehzeiNw963TnkIhxGWhmYMqa5_bNYKlR5yqiJVbyCcJnErua0JfSg5O_OyultevLIdz5XmHKFBm7lzStZHVobegTRXLkeD1ptNZhtwJMKt1FnQMggUKMGS6loxCXo4j0GhgiYEb1m3MwRhZYpQcVm6pYhdBCLiVQ98CnavtWUDYPfFWQ2uMpKGmAJ-Ei67EMyJDKcpGYIZ4MhFMNkFBnksoqVMkbO985BFw7ZChskG1LH3FhUEEA2Eacs1j3195QFAfNEZIAhvtsoY5C5TKLBxfIFTiOHDEnL29WAvDtsHuPedeySBzvLE32ysNLuFOLS1TfaKw6qd6J9BmnuRbb7cqSRxeRRw461bVMjwX-CHZeV8qxrJTuIK4j8HJ25PQQ17IpJpNYeTw1nkmFnAdUfai5IRYbxCZgBKxjwBnes6eA2lxz8j6_NnlReBdyWodFR3g8WDfFPmjf1-Ab5jjICO4Q8iB0x0OD73bPf_ZDJp4uOWklPtTYEeKUIHKUo8eQbcvGvp9EW9BdpHO80jQs4fuTNdDhZPbExYamt1Wjv4S-I4eOGVXVQymkxrQirYyQ2cLN4mjJyURGj-q6xZn00BE6qHUeBWE1yw2kNBZ1_Lr0-CBQGRkk0CjUQCZ_D7k2PumALgJeUCpdWjnstpTIg37qRICgFqA5Pk8FMjCpq8CknZ1cCqMbZu02MzDQe0SXoDZmRbsJ9uJKwIJK7T1rHp0BdfwnscCkR3xk6El5cHxLpGG_dLOMo3sp7xAEt_-tFebrqMSKHjkH6At2JFWii-dC7uCpq2wixyQVsYIGmNKuIOLHDZCvcwyeUdB_o1aFMMllFrkuFOUPXUswx6fVDQftwyDrRcQ_X7Eso91F5uGYJNC-3qute9G_12FdSyt0ipr0myd06uemdm0aWok6u2MhFxLYtAgmY9SJpBsa7WOKmCrVVR7FSugV71u1Wt5gCr8kfZrOBIlhSpNqQaotL3oVD-BSOeIo4J1m_BUNacQL-nv-6QEZeAsu0A0Sc6VgWt9G6u7-La17af1mEnlCOSTqtj00IWYPX_PMNSzYujIlTlQ2PjibLcY3EBpb9V03eGwb-bScTWqBdERcTCBOnLOwXAbXbPUk_PZ4LTtygGX575n8nVXgMEZALd8iE2sbJSYDzgyHxvPej5Zviy_ojkXuk_SXEGXdWQlAL6vNWORz8yqYcGV5CrAuNppJ8X3t7yIUgNngWfna7b8hu3AAXl-teXevZrYmhbzY_ruXFPQoXa1i-8TyOk93xIby6LmydmRMItFMoGUuu4kgQexpKv4mH89IXCnPxZW-amseHO747eGicbA1Q-igbNYYvY0HeJuQnsvgJoDVVLBYf8ewrKVARwdhaZEk4sVQXqw8IPYYytcEzJM482ovD5nFB1wZYsCOYZcSGxgxj6C8t5GBMrJ-kvlTbPIRGpNYvkCvbOkC2RZvlTqpDdqbpZ8UiFPQJGFYwtyd0ZzJHxta65DP8UIxDws_ZoEQcSp0Bt7im8Dtje7woMcggl5GpJQyN35A0Fs8USbiQIRRw9gEm1wtVbDOmG7lqAQAajXPDjJ0cobVIQMQV7XcA5y_Aw-eTpPEpBuV6vk_Ez05k3GiNFnsyur7izJawr_QGR402g5qF_nPAG6Zck01UAsi5N1fQ7TajaFPJ4WhvWm-EtlK7hFKtyyf3LEMfIL7ksNjGS7Psa39CiSL59kwGyrpptnlQ4QZUhTRFAZW0HNNK5TPiCq9YjvBlrZwUReZlwdob-5S7pCZDmRH3iYQuIMNaCQjCxj_Au0Rfv5BXNga0x2Y1iv4LO1Tj18xW89p6M8_NIxRDQJQw2iCQLx_VwICsiNEyynenBICgojRCWt5AIGWSuG9mx8iMVRjrX22qK9CACtp8KlHTTI10CFFVHOmg-joqldS3lRYljMMcfIfVcjl3OaENrww0hlatuo0W1LtT9PrL1lEmHY8jZFMrP43SA_sO_nlJ7Sbult7Mf7Bx8JkddIs2K9WjPB7mcoQQ41tqutStW7gylhs40tUzvz_eQefxszU6-pRaP3CVkdRGHI6Uggq8_OpFRn0oWet7ovd4kirSmPbwPvEQPLeHt6_7JXC32TrHBdSBxxtn3i993lH7ik_TqcRgEz1Q==", + "summary": [ + { + "type": "summary_text", + "text": "**Calculating terms and sum**\n\nThe user asked me about the 10th term and the sum of the first 10 terms of the sequence defined by \\( a_n = n(n+1) \\). The 10th term comes out to be 110, calculated as \\( 10 \\times 11 \\). For the sum of the first 10 terms, I added up \\( n^2 + n \\), resulting in 440. I also confirmed this by directly summing the sequence values: 2, 6, 12, 20, 30, 42, 56, 72, 90, 110, which totals 440 as well." + }, + { + "type": "summary_text", + "text": "**Summarizing terms and sum**\n\nI confirmed that for \\( k = 10 \\), the calculation \\( 10 \\times 11 \\times 12 / 3 = 440 \\) is accurate. So, I can present that information concisely. The 10th term is 110, and the sum of the first 10 terms is 440. The general formula for the sum \\( S_n \\) can be expressed as \\( n(n+1)(n+2)/3 \\). In summary: \n\n- 10th term: \\( a_{10} = 110 \\) \n- Sum of the first 10 terms: \\( S_{10} = 440 \\) \n- General formula: \\( S_n = n(n+1)(n+2)/3 \\). \n\nThat should clearly answer the user's questions!" + } + ] + }, + { + "id": "msg_0a7d8cec3288e849006a8404a7115081999cd01a44b8181cb3", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "The pattern is \n a\u2099 = n\u00b7(n+1) = n\u00b2 + n \n\nSo for n=10: \n\u2013 10th term a\u2081\u2080 = 10\u00b711 = 110 \n\nThe sum of the first 10 terms is \n S\u2081\u2080 = \u2211_{k=1}^{10} k(k+1) \n = \u2211k\u00b2 + \u2211k \n = (10\u00b711\u00b721)/6 + (10\u00b711)/2 \n = 385 + 55 \n = 440 \n\nYou can also use the closed\u2010form \n S\u2099 = n(n+1)(n+2)/3 \nwhich for n=10 gives 10\u00b711\u00b712/3 = 440." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "reasoning": { + "context": "current_turn", + "effort": "high", + "mode": "standard", + "summary": "detailed" + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 245, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 855, + "output_tokens_details": { + "reasoning_tokens": 640 + }, + "total_tokens": 1100 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-863a18378a48.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-863a18378a48.json new file mode 100644 index 00000000..e6c9ece8 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-863a18378a48.json @@ -0,0 +1,48 @@ +{ + "id" : "a5fb65e6-7334-3418-9d94-15a9941968b7", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"o4-mini\",\n \"input\" : [ {\n \"type\" : \"message\",\n \"role\" : \"user\",\n \"content\" : [ {\n \"type\" : \"input_text\",\n \"text\" : \"Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what would be the formula for the nth term?\\n\"\n } ]\n } ],\n \"stream\" : false,\n \"store\" : false,\n \"include\" : [ \"reasoning.encrypted_content\" ],\n \"reasoning\" : {\n \"effort\" : \"high\",\n \"summary\" : \"detailed\"\n }\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-863a18378a48.json", + "headers" : { + "x-request-id" : "req_2374fedd7c5c41e684b0382260b0e69a", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2cf14200a58be44-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999752", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Tue, 18 Aug 2026 07:07:06 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=g_swnClrFoXLpDVkA2hC3SmF.sT9QWZr1x6z46ZFhw0-1787036815.3688374-1.0.1.1-KBXysFROvaa1JGRl_pdGPQrwj6aUsvdxpqocRFKpTemO_ukF7t7fVELSyBT4TyUVfUgOUu1Hjmw6KQRoHBVtSrR62LcqHgXmy_ah5gDs85F8TO9PJykpTUc7E6Z_udls; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 18 Aug 2026 07:37:06 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "10743", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "a5fb65e6-7334-3418-9d94-15a9941968b7", + "persistent" : true, + "insertionIndex" : 44 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-892784bdb435.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-892784bdb435.json new file mode 100644 index 00000000..48fb9acb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-892784bdb435.json @@ -0,0 +1,48 @@ +{ + "id" : "bbd3d5ab-69e7-3a37-9c5b-ff00f3acd4f5", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"gpt-4o-mini\",\n \"input\" : [ {\n \"type\" : \"message\",\n \"role\" : \"user\",\n \"content\" : [ {\n \"type\" : \"input_text\",\n \"text\" : \"What is the capital of France?\"\n } ]\n } ],\n \"stream\" : false,\n \"store\" : false\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-892784bdb435.json", + "headers" : { + "x-request-id" : "req_22ffa97bbf4b451da6b6d56a82afa083", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2cf1c6b2f95deea-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999967", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Tue, 18 Aug 2026 07:12:36 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=8jkAhqYp8IHoaAQXmFcLw0SamiwfdW.Z8NXSH0VyqaQ-1787037155.0670147-1.0.1.1-p7vFikQBPjTSkS8Z3lSFrC9SAObrNdPxp.9qci.9PcOiHCtR_bNapNGd5C78zvQYPQom_vp5GvZFidxAxKjZ22n3Oxw3MrOCi.8piOCRTw_4I2PtF_kgKagtqahgGJ6X; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 18 Aug 2026 07:42:36 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "1000", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "bbd3d5ab-69e7-3a37-9c5b-ff00f3acd4f5", + "persistent" : true, + "insertionIndex" : 45 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-bdae47d54959.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-bdae47d54959.json new file mode 100644 index 00000000..a79458e7 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-bdae47d54959.json @@ -0,0 +1,48 @@ +{ + "id" : "3ec1669f-6378-3bd4-b5cd-dc7acc2f8acd", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":[{\"role\":\"user\",\"content\":\"Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what would be the formula for the nth term?\\n\"}],\"model\":\"o4-mini\",\"reasoning\":{\"effort\":\"high\",\"summary\":\"detailed\"}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-bdae47d54959.json", + "headers" : { + "x-request-id" : "req_b5b9fa2afb1f4f76b9dabb37ded1242e", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2cf1426cf0effeb-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999752", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Tue, 18 Aug 2026 07:07:07 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=WyDmQlMpNI946rkSfP1_BXgVn1_2srN6rb9vpHMKOM4-1787036816.4511461-1.0.1.1-vtQzH_l2P1CsGkCSKn3p6wJ1NaMe1HllnqPd3D23F7DzNwuuujZTu4.5zgHWAHROk1ZGez3ncennXvnk0TdKwTRw64SmrML1axFeCiMuxZ8NaZwj_M9KEwwlPxqK3Sw.; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 18 Aug 2026 07:37:07 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "10799", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "3ec1669f-6378-3bd4-b5cd-dc7acc2f8acd", + "persistent" : true, + "insertionIndex" : 43 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-cfec8f65e9bb.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-cfec8f65e9bb.json new file mode 100644 index 00000000..82c29670 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-cfec8f65e9bb.json @@ -0,0 +1,48 @@ +{ + "id" : "f1fbe0a1-942e-349d-9d23-1b1a952d3336", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"o4-mini\",\n \"input\" : [ {\n \"type\" : \"message\",\n \"role\" : \"user\",\n \"content\" : [ {\n \"type\" : \"input_text\",\n \"text\" : \"Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what would be the formula for the nth term?\\n\"\n } ]\n }, {\n \"type\" : \"reasoning\",\n \"encrypted_content\" : \"gAAAAABqhASa-I4VIqhY6KSx4kYjkcZXA73TG-7gcpuRhK_jeMdohLuEsGirvpCrRf8GNycUeIpo4BC3d4LcuYGtPEn0W6RlK44EkxuSQCdVLCbSxFyf5lXEpOGS-DOvK7Lya5d_1wxnohQVGebu8G_Y8PmwoJS2VBLmmOe7PIMJ3D1VMyWfiwIOwnVCr0z1jOBpvAGBRPjn2_4p4qN1PotDmkKHjlVAVWIExZOXH0QViNuH5q0wpqWgpBVwFPAtF34hEpeD7XKato-CT7TYV9aUH9m9qYtR9vwqkzg6gm6z43ZYtZn5ZNoTEJ2ECaO_NfvKmn_Pmq3ttetUD8_jiD1kvdu8B9fgOLDtOTNYuEePht8rjS_ddGEKysgp2UR_Eul5eB5jc87d6gtgcxz1226d3ytwtLGzDQWgq3i7w9ir4JE_UBJTwF0ReWjscNEkCVHeog6n39t4R1TpRMbSUOcnkjNZUkJSUSsrD8D7CFzyXgzaO5_Tl7Vzg6I9zQrKF-5Y9ILxm3iwQ74v4X5kgha9jCUL2JZkUXb26c44FxJvVDuvcRGSLNjIR23kEiM_p8sWu4qZjVe0GmPhILJg-CkOmUZDbQnjRSPKYqzemz7mPDTy0wlGf-sMciwedknWxD6HKepY5kE8waLyM5Q3IkOIi30RRAVLGzIPc5fg14NmzF-KFhpK6yEfhgsWq4LQ0-ipb-lFu0WTtsp0KoXqO-wqgk-cPxEtf33wMlmWklNJ0FeDQxrj5M-JKTurZ2tqCwYpfrnvL9DpdM958NSt4SId4k1ybVEqCTM-bc_JtdaxS3Wr34DAw3X5WS-b480HQUGCFG69QF3mku2nN6HlN4TTqcexUfDFH7xOLiZC_uYVh6zWuEypo6OC6rb7YxWOJqJJYLvvBcq6bJuypMZfxMdlRkfqG5mM0a_Jeeka7NdSlZA60sATxQxWiMikaBw5H12_hWSMzqTzgsaesjroQVpHEsfKEyK1UlyXcH4FiQWR8eZusGoq5vtLqb5bQkNoJ0xRkDYop2H_UefS-07xvdqwNPDocpyxwGaseXwLK3PnYdC4OlyEjFI21HHvDU5ET5od2PWqyOd0jmLvmx6r4Q5OlBtvPse8C1fIjq7oPbCcgcxwzAJjVlY3PBnbxDuapP2BPFV0PkLcGCMDE3UcLnQZ7y9cTlIO41KlNtAFEBnl7U7NzDpYKJY0UslVVeWRU3ulUBoQF3VYB9e1MEY0-HO11hf7P5q9QNATH3JieSXGvGNvrr_sl8nwM1rzPomU1P-m1wxGxtDZ5RADzU0WxFsUZCBY08bYaQtIIl9Sue7PzKRPWUTMxLG3dBSlo-sZi_10j7iO4JbDQXSTZvPa26xopweRMyMNHnsB-REu3xRXHSEjv-fPnJGFAt8gURAxW2nv3LEWNKR7WX1phCrkQF6IH398MBtqMik6oSSQuYHs_gg8BfxFGZp_RMKonTIBAdlNP6dMJURFfYIeissaXNrozorRSOdAzKOxFMcI8De7L2eluBXvIm0ZArtm8zM21XuwnTPvHC0u2FnAGJVQ_vdDjfnJwGEX-TU3tv25qTwvAxfDKwkdEacuK5E_Y25uTdP1hLd-LsLqIxWxzx1B2utozU5g1SLb5LmXiCKLrm4yI7qNZC-TL-CNDD-c1PS2jVWLQhaU0AOu3g0PSnBnYOEEou_o82oK3Bfgriq9ThKbD0BZGPOWEcWSUWNe2P_dwDAgZprbJmkYVlBtYzpSgwC7m9oND1IOFBVgRD9xlnpbv21ZHU58CXnPVfOAF63wv3bH5v6kU2GaL3DbBQpGIbigAV8lRMkNySsT20zVU_5ezh03xw6ZOmnsja3wAKCzCoLKitt5PogoxsIBOVLT4r7_rvax4JZ7EShaQ_SBiWWxNQ6LXtYKsGrDVC3-7ZTao4PANZSdvHgZJfo3tpbxgIJx0MQ1Y-l-Dptt5_NdANECJ2Ro_buPei1o0jAweU-Vtx9ofT5N246wrJFY9Ejf1b8CiRGxtRWZ6Qs7qgwrnt2LbRXAyNYuZgItuX1wj-nKupCe_BB8xUS0ftDGuoUl6OKc6NSCOHpE9zVStB-wsnij3j9IoETfAjlpHYOza0zC24_NqQyALHqINrGSsyVcXdxtwXBVcOrn4twOMM5h6JqtN0dcRzZlNrvmcHHOgbl223H-5bwbezXyTFjUvLSgiLGpvemp0-vSyZYK_hnXHdFGXZFGIe8nqiLgp64RQ9qEx12CcOqSwnhmB-QUIy-JeUth_Dxq2bS_zx74IGd-sIEhtoAQx555Sg9Cf0edCXlK-olyGZB0FlG2ousWvK-OinqiltAn-ZCupGSuWSwt70wYSIhW25Xxy3N2Rja0pOsVYeZi7kpRH8doZW3cG_ShlxqIHz9g6zSgKtXHtFzoRzkrIb3bXRgT9FGDjP15WIzna0u_uj-F8P8Msi3a9J-Wdsm_xlpFYPd61xDoLYspdpiNbE0OW5krAp0M_zkr0ozxpMdNPnAJ8WHPNdzZJJagP0qaUCQyu15x-s4gunrJdfVt_Y8MB4U7qdh7u0tynx7dvL1Wx3mYfqyY-ydVTBn5PDrrZ7TRb8AGDqBtDALfaLanlPyAD0vephy5nKJFHlL8VZgLp5Yd3NuX7GN5TuwZGRwIhpmxLs-Hrdf5hllqXHyoG29VlqhdspJhzPtqxHJJ0AIG02JmGrrLd3azVYNzPes4JL7smyWFNA_7dRAgDrg0HiEPZm7Xc_ZvMe8ryCuN9ijQ3FG3WruF8_499EK_Be4460yomt73m00ZFV8So4UHw5CJMIF5ju2K5SGrqQEhLnTwhsp6fWTqqhMAO_p3JfWpx2KnBzF7eZfB6_BM_jbYzfZ8SmEyInbZ3p9-BPWJiGAd551nehYxGtQreLSu93rxgqmOvTaV-sUAX_50biJUJ9dh1DA-qms5E4ktB7GvXjktJRJrD1WaNMMa7fCMpOLTtPCa9WoAE0I1OLKVS7znw0V3-bseu7WOB3XWouJZWxfX2Rx9vqqX0iF3SVoz3Ot1FebYu45kOx9DszYQwLYeIFYDTabZDLqTKjJ5_dvt2YbPmLmkHItTpdBSOtMwKnPKVMdZUZ4V1CV_w8mTMU0ElJom9sIjPY_-A38MdtZzJFaz0zmEEBeqFQJkckllKKg7uZE0OnVUeCqRpUMrUTtRKaYFns_mM33pjBQDBePhJ7DjCGrlxl4B1_lR1W1PeALgKpErd-EKCnXltXSbTKrqwC6RiCoGsJOVdrBPTL7nAMTFYEOFpn4JarmzZsq2-0Nz866PIWQpcICpe6Du9oumr-Wg9ARMp0f10HnPAZm3GcynE9bM3bs2rvrmGbRAjxR9ToqzGdmXDp1rrRsIkyEFmFgRj5Bc_9lULYhC8Sf84GUObnVsAHhQ1DBCN8kOPJAd8aP7UAjqLAhlC8hsuhxNF3w43GV04Diz3CoCf3h3kpSZ_LAykxSYXZYb9NOlY9bgLnFpJBJqIqlpxbte2APqdDTxazz10ZXN0zZT8HohJRhLXqqWntrnFp7_em_9FgWGcOgYHkYtu5R9z7v1Rcm6d80Sl34Wn2joPioBGUQyw0ksQgS-R_JYZwDZ5Nwrq7CY6lk-amfnwwm0JkllVDVcXeelozumxQl4zCtLZ6taAwPLskmIoyRH02Ko4rrG9ORr6bkC9DfVq3jxqVAuLSkwjpTKxB7N0KiEkOq2gGoHvAmu9On5Y2o-SQbqy22ngXYUV38hHpElhn6ZS8fjo3Q72jq8HaS1pZG5jdlwGPHaafiBHKmtGMAcQk0-4aUhvcNvJ-Eyvn2VJ8Gf6nUS_5Vxydtf8KwEzGkPtXg_7ktWzHxND-ALYr2W-GJDyj4y4JYgIr4ikwUjdmFB4UmX-fnw6Nm_nZFMOTWTD72sjBoXzoYFRrp5RTY1uXLUZvJt9FHYSMBW2duAnGA5PO9VOKGIcWhF-Srq8SfHFXDhlky1ADh3I5vvD7sX-R4jRChNqv2olL-5G72Kv9Mjt_-5t6Ax9FdSkYifz_C6if-t19XyvuzFD5a6MzXi37SX7SgAYn65FBbaB-smtHLCSZkoFlh_u2PNZ_6yVThzGo-YSfSe0wN4Dxpfw8KhREcnMjN8HjvKT4Ce-viNxPC77VDYHPBxulFxmtEWn7pZVoBuDOXqzG4hOWvdQCHVyGBSiKVX6DUJiOdCnY-anKgYxQldy8-tULNMSLYNvTjsjxV0RctMjoWdkP_WHGZsLTtYq8lqswa3WN860s3tJzREucgH62FlNmFPqyXon-cbjk44_lbGlO7de5lZbW50PUqzvGjkcSQ20yOfsUInoexh70JOxLCMUw6L8fDqabSmOJ45LoOi5rGKRHN9avZ_kLdVEm13_ngX2KqEBGmocQa6rFLv6nQIrS82w3MaG7QYSR3WAYyYNKUahbywMDdWnzx7NhcG9u0q4V1v6mpm0x7gW6rjgi-7Dv6KjiSI-V1MYCU8F8DefIi7kekLNuB4l9BLrd7xwu2hKx23UGu38eZ-B6eNKiactA_G5t53n8w8y18ILsKGAAXlMNv7FHVfol57fWQL9HzPvXkC4TfxJ7aV3WNfWuabKFUaBfdPrG-bWEdfNNcnO3hWgeTSqy4Tns2UPuXTk7d5wWqBl07T6D_QqPWs9kDDBVDLZmjXDLi5sVg8PN_s0uP4C3y_lTb5VJao7LjpPj9wXRKg_Vb5TNCo4t5Apjgh3vT4qJw0RtI6mVMuCgAeYiMdq2XjasUdUY9CH9U6a1M4IGn0pRF8Xn822B4YHwxsCMl3-4i5RrbtGO9GId7mT0YELi77VJg7MKA5yiAI0oy8n7xcjh0tZKa9n_a5h3vH3rUOZ2x3gvPy-JG_3J6UENs7oSAfASxd2W7IymdToBP2zltKn-lq6gas-pVCFy64ghZvyI5HTlFlxZc7QP5hLPBFmJTKTbYWBxRi7IAWn5JY08QWFpSgCnZG-PDEd838-RxSWIUj3wyJ70nnvjoz4AVPL8gTwu9RZ3ZyPDD4yxVV1bpWw5990iRtQztsDpeEiQ2Eea1Q1JVHcMjJx3gGn7yWUOwFa8InevH9sAia9D_HYQY3oIuXe3DX-pdXCxMdWvLv_JqpuU-5Mu9UjXxHmABAgvjHkykWwBsYVGnGZcGWAlpaoLkkugk-Wk5Z5jsIppqsH5KXbVIk_MZIG_glr-f9bK_3SSY_bGbglV1KIYArq1ombToAe-m7BtYx_YJpXCAMRfltSclZk6WyN95B8SWj_d0UnrUKXDUPwtK3WiIJB3ao8rSfEpn4_46-gYnAnOwOnC5Ss-8t1PQ4gc3cimk9jmIhbK95Su33Z6WevZlzsyLalL3PKQR-_pBs0txL3oVSxxueLdp-mCDtdkKYyZB7Ql5HGfqvvEfXGuBjlaJHH0oKAaf1j0pnx-PK399ggDEkyNo-Gz2cqv_0EeauZBOivQugCKq6P0ceEf09_eTajzb2xfcpEtuqe9vssNxEeIr4jMucTf0_heRTbcXjlKEPjsmDk8PpqvDF5REdPVOW0c8f6mUTN_6d5D2UTAH0K886KjMVFLJlyLR7LMtNwGfpgjlWxlF22riTmGr2lKRGBj_gX1-kttex0kHxOJtC3-jEZPLa7svAU1yNT9U1fNHJbI9jsJ2JeF_8CDID9uM_MhJbXmWAa9AHBGWcpXtPCScAJ55vOcXmLbKgmoBy46ROMMH03S0HqdOLwwTMMdX5KBCs98GpbXAtz-Acr1ueLMuOXiNWTjKvA1dqN7Cqxus5gJQEvuI_yS6oeiVlGWn0Y4z48kbthXl5Icw48XpogoU3x_S4-qEFf6iYBhHImz9dCfsQ0pWNBGF3EFsOISD8xgCBN8-ZThpeew-qF5YhW09A0mVRpseOJZtnbbDOmvED11TI--sSpyatqVk-bov5VG4EqFzr6VZRpr4lemodUL1uZXyP7HLH4ZXeCvPTDE3qjHcuH4HvSH-m0WPvzNLlmEPOODt1Tnw-3p8LXGVYJbU97-zLpmahT2c8CIxZeeg0OOgxCJgL_LtzALfOXFEmawVWzm9BDJwaOTytMBCXpDd8S8y5p0mHJkiaVp0yfajjUu4z7ZBRV3Fa1Z6exsU2HoUztUo_o9T7MSTn-tT0Bz3HLM5DFWj6oB5zOvHYplqHSIWsOmGmezN8iI_5B9Lg_8CjJ2WkbsimLcuPnCRDJf4cgKb6B6ftA-TIPOJsBq_9Ahc3qW6WWTO5BNR_BfMDxEwNTagNWjA7V6WGaaO56nzwmLc9GdTazx_7JAiTaUqenguBYKzfS6dvuXP68DJeq6XAUBPefo8MZMtqPl5CjwSNrQjraqckvolWaRzRZ0ClfoAXPWwAMztJo1jZv1P-KN_2LqAx52MKSQ3tVv03d_2u_tZuZeDOMd1rN08zvgWkgubJmwV_91Cl0AKwnTlOMySaQ0thwUCeT-bz5oGwJOdXoVr8247Nm0xNxtzvId_QhXZ0snuPnHDSNw6Paz9x9FxoQ4nhS09ep38uANyuwFkaN2dUnVtLvt5m_-8pw5iHeCo7hpl9P_DDsUTBsRHG9OO_QirFCSife6XeEvP22yi3IwZj-CRJ07YtXEM7ygdsu-Sj-ATzQFeSulUixESO407jLTlpPT0GGqQWSdVaSC3969ORZsEMrOoVAbEnhmpr4c76n0fzzlFSLETMHU_7HdfrcNKfqqLtosBOzYPonieFNNWGPrOsj88m1ei-ENcuSsaX_8k1tkZbnAFnSEt0gqAlkmpj7PXILVnaGdgemi3iLDd1L4PLbtHkqZLtzNwyAuUoiBE8EaUAijF25A71XusjB44o0bVfg8ObBbvTkKf9_NgyPCrpUpYs4bEpEfisOZ1mqezThF3862xKPWQD8XKBjBgMBe9xoPq582wtbEx59fhLaZQ=\",\n \"summary\" : [ {\n \"type\" : \"summary_text\",\n \"text\" : \"**Identifying the sequence pattern**\\n\\nThe user presents the sequence: 2, 6, 12, 20, 30, and I'm analyzing it for patterns. I notice the differences are 4, 6, 8, and 10, indicating a relationship to triangular numbers, where the nth term formula can be expressed as a_n = n(n+1).\\n\\nThis suggests that the terms represent consecutive pronic numbers, which reinforces that the formula accurately describes this sequence. Overall, the formula seems simple yet effective!**Establishing the sequence formula**\\n\\nI need to clarify the sequence starting from n=1, giving terms like 2, 6, 12, 20, and 30. This aligns with the pattern of pronic numbers, expressed as a_n = n(n+1). \\n\\nIf I were to consider zero-based indexing, it might yield different values, but since we're starting at n=1, I’ll stick with this formula. Consequently, the answer is that the sequence represents products of consecutive integers, confirming the formula as a_n = n(n+1).**Defining the pattern and formula**\\n\\nThe pattern I've identified is that each term in this sequence represents the product of two consecutive integers, described by the formula a_n = n(n+1) for n starting at 1. If I were to index from zero, the formula would change slightly to a_n = (n+1)(n+2).\\n\\nThe differences between the terms are increasing by 2, confirming that this is a quadratic relationship. So overall, the final answer is that the sequence corresponds to pronic numbers, with the formula a_n = n(n+1).\"\n } ]\n }, {\n \"type\" : \"message\",\n \"role\" : \"assistant\",\n \"content\" : [ {\n \"type\" : \"output_text\",\n \"text\" : \"The “hidden rule” is that you’re multiplying two consecutive integers:\\n\\n 2 = 1×2 \\n 6 = 2×3 \\n12 = 3×4 \\n20 = 4×5 \\n30 = 5×6 \\n\\nHence, if you call the first term n=1, the nth term is\\n\\n aₙ = n (n + 1)\\n\\nEquivalently,\\n\\n aₙ = n² + n\\n\\n(These are sometimes called the pronic numbers.)\"\n } ]\n }, {\n \"type\" : \"message\",\n \"role\" : \"user\",\n \"content\" : [ {\n \"type\" : \"input_text\",\n \"text\" : \"Using the pattern you discovered, what would be the 10th term? And can you find the sum of the first 10 terms?\"\n } ]\n } ],\n \"stream\" : false,\n \"store\" : false,\n \"include\" : [ \"reasoning.encrypted_content\" ],\n \"reasoning\" : {\n \"effort\" : \"high\",\n \"summary\" : \"detailed\"\n }\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-cfec8f65e9bb.json", + "headers" : { + "x-request-id" : "req_0ab7a3184e6f4dc8a12c3413f79a8819", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2cf14652cf3a341-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999612", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Tue, 18 Aug 2026 07:07:10 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=S4jf0c57cLFi9rxgCelwwCE6Aip7_MsiSPtKtJ1Fd0o-1787036826.4278164-1.0.1.1-NAt4t9SuMKiqtWqOMgwsDAcqykwB91rE8RZIJxkfy3cM37lQmZD.OQgLFTEi8risBSjuVPsQPSl4C1q2BK5P1w5A8Cixs5EVdvCwiBlWC0dfG47qEmbHQObSIKnb2pnB; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 18 Aug 2026 07:37:10 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "3996", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "f1fbe0a1-942e-349d-9d23-1b1a952d3336", + "persistent" : true, + "insertionIndex" : 42 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-f0262cdce49b.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-f0262cdce49b.json new file mode 100644 index 00000000..6ee32b13 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-f0262cdce49b.json @@ -0,0 +1,48 @@ +{ + "id" : "72ab105f-6109-37f6-9e73-8e7837ba889d", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":[{\"role\":\"user\",\"content\":\"Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what would be the formula for the nth term?\\n\"},{\"id\":\"rs_0a7d8cec3288e849006a840490dccc819996b53c37df120094\",\"summary\":[{\"text\":\"**Identifying the sequence pattern**\\n\\nThe user shared a sequence: 2, 6, 12, 20, 30. I see it follows the pattern of n(n+1), which checks out: for n=1, 1*2=2; for n=2, 2*3=6; for n=3, 3*4=12; for n=4, 4*5=20; and for n=5, 5*6=30. So, the nth term formula is a_n = n(n+1). The differences between terms are increasing by 2, confirming this pattern. I'm confident that the answer is a_n = n(n+1), starting from n=1.\",\"type\":\"summary_text\"},{\"text\":\"**Clarifying the sequence pattern**\\n\\nIt seems the user is asking about the pattern in the sequence 2, 6, 12, 20, and 30. The nth term can be described as the product of consecutive integers, so a_n = n(n+1). This also relates to triangular numbers multiplied by 2. The differences between the terms (4, 6, 8, 10,...) indicate consecutive even numbers. If indexing starts at 1, then a_n = n(n+1). Overall, I can confidently say a_n = n(n+1) is the answer.\",\"type\":\"summary_text\"},{\"text\":\"**Explaining the sequence and formula**\\n\\nThe differences in the sequence are 4, 6, 8, 10, indicating that the terms follow the pattern of pronic numbers. This means that the nth term can be represented as a_n = n(n+1), or alternatively as n^2 + n. If we consider triangular numbers, we can also express it as double the nth triangular number. So, the formula for the nth term is a_n = n(n+1). This captures the essence of the sequence: each term is the product of two consecutive integers.\",\"type\":\"summary_text\"}],\"type\":\"reasoning\",\"encrypted_content\":\"gAAAAABqhASbJorYQo9xxCWixpp9X6MBEKXknGhPaV4Y5di9qeYJ97cv4Sm8m7lgi-GxF_tADp1kIhJUnLOKGZFA_awKYpupqjU8hQPPXq2l1sK4tBNmGCPyo-HxoN2ZXLNI-8JJ3QrbQNoBZKSjF3YzdkJEoqBnKec9jcxDP9v8toTCw2JRoJ6QyqTOwc_y2_8K0qxv1qSZLOokxQ67uV6prq55I2m4LFu033ITv4ZvEqZ9DxThDpwjYTzn_tkgR5NGZeYxsVO-oNOLTD3ww8a7y3GbEHKQhTF9YU_Mh6rLQYdbkN1u1HT7g9iYzhz5uf4A7CRkdMheW7Iv3onS3HtdS0YwCIwY9jZOU0IqDSkPowalTtUWAt55_SDFT0VLMqUSfbITXuhSQ-8NwG4J81WDYgu9o-nQ_uJkodTo3WRK6cRks6cqLfdjhT9OuiXVm7y6fKUsSBk6Br8j5wB7b6zDa7xQbxLu6G2SaSzG_EVURR-GMzBD51FBpIFq6qpx9rkvFI6rNGSzkj_kX3pTJYQyDbQv_y7xOEiC5a7XVtqSFM9Ng38kNxSaaHguV3SA2rPRRKwbjDpCK5I550Qx1XB7Xsc3i8QfqofgcHcVzGmbfYLCzkNQgtN4vMj5Vbd_p0na1-eVUSsYnA0ZcPwqrDBHwus12qJXGHkblaMb98h6D0IGZ6FIq1Yyc1i8SGKUWu1sWGJSwfcXYzmadRMjJqdCa_F5IzLu0tJVjqC5afxgr0_pPoUL28hc4cmi8QqeW4M5XLwVeQ4i2zsToy1B9zeI2IK-LcGiauAGtkwn4Sqa4woWH3KC4onIBcFN9jGVu62Ati3OwTACGVO-u4x8ZGSMpli6bI_4ihn1nk60I_8v6wSuziP8eOnYhwLL7JazGGQrS8my8mhpDdjxPFl0SK_6W-stnnCpkgg3tq4lu0vlsGPETo8rGXbRoESsw-Y8CiGtqCLki4ZIAFqJ7HbaJcWPFTKgOcoGTq4Z2dOKZD3f4tQzitglg4EygzSjBW5YPfROS6h6h83Xv56xfL7cfO0KOKlnNoZxsdXVg1QCMTyxNTXp9RhJHqdH6VCC462fR2ETTWE2xoszjeP72evQwE3z5NIhUPmx9UOSFMajmZhE_p_nAPunGD2qiUyU3yBmPNh7Ko3wj_n89v4P4VnpB1sPA4JXitcBNDdRu4VjwZTkSfmhbD5yspvy5I64tqQ2BlJDfn3-B1JQDUur1ihDwNj7bGWwa42mKfL_GHr4OrFK8J8l-s3hR75HLya3cGK9el2fEO-bgRWCbGI3HMMiZyAQHzA4XU7LK5HiUe-iNZumT2hDP52-XCffIRnivNL6wJVsgd2EZmntk-TRuaqnL__TEkzEtcncNactyX7KAHhfMpZIdUY7tMKRWNMQWfMNBPlHFHjWzPQXjvTAgKdsCc8ob4eljfS6VF9OeaDDe_BnNSgFk6gCyWZm8q9plJrnxhYd5Jd62xxpdS2fq0wO9lhmljitTCnILCCeOH6L5gD-1xh4JlfKcqAvTNtsBG759O9l2v1l0z8QeWw5neBhXTjH5PDS7cpqqmm6r0nIT3BwfBxh8xgZGFMPo1ErN1fmOCQ2tmIB9YGNYF32V-RoluaTwL9Z2CZMPB5x4ndaM_JK26_UbU80wsrptQcmpikbP2A1DVVCk0E0YykLmTUJknGN5VDjpSd_PYAtOHw4a2nXzdEP_quE6HcE64jQRznRChVNPUDOTW7Uhr8_V_gWoC1sVykrYmMsFSE1h6_1Bzd9o-OJg8z3IfZLr24MDcBBnR2MOPVuPz1zZINdsRICTOHNaLPg7AakOO3IPSrKJMLkJD4orF8mzaNnUIx2QuZMuKIeHWdL4e-eUsyfAxKvo2q3WYjOeFADz8elkgDKMjJkwzop8LYk33C9yrxGYh58wlxyLIoAiRQODk6y8YfNk6QNdgeclESjUVFKapGtMvOida8QHe34NSH22nN5oMNDjigYIGqLAH83ahz3zl9UNwW9Z1UDyDkqVuqao7JGQFYVlgo9fQLZEpXH4Eb_m1qFpAoyGceRDgVpg3JuFM_fmtx3xwasx0YmdVx14_aVqS4qhg0IX2q6RQkfRw_2bSaiJZWtjnKUCePCNFu-HaRap1GaUbhzom_aYSxIQvELPdhEiOrz7BffoePstMeqD_Mtq7Ow03XnC8be6Oiqrt43ndq9k9W8QaKQJRDtwZ-N-PvmDoqbfF9Zi984bsAcU-mA-n6GsX7UxJ2Rfo4UvOHHvEZUClD9XhMv5ov4VPZATguOGvEYe9EQT3MixC5rIaNGV1LyO61G_q-LdmRo2uaEF3dV5UXttQkyWpYRQ26sgB_qoZ8_x-MS25GD4KvQkBHDycoQGkidwgR65DXTlT8241wB1fu-tHFCxPqnbxRqgsOttqvAUJjAffBN-6lmLBwBLzcllcr0MpSvKYP5aqXgrSPBKWJvcV820F0sc6NSVNn7Wfs7euiCxfLtrt4-Abe8ZaodKD6oaqOYDHmdK5RqwtoU0_dFhWf3_z7dumFjhBo5OoeRHCMDmw5kDIYRuwwqU133gsnxG2jVHObiUMFZLDYAZ_2PYYGsovAjCKSSFeIcg7Rkw9LYiHUHmsK1AgfORvjz354qDTugoKfRgyFpZKG0NWmNcv9VpUHI5DL_sbQK5qJ7ru5InyORURvK3xwg-JRgO3UDKc_B_8TtQMEpH9VD09ihm8xSD-zgG6_QLCds4U3oZilA5Mp5G_szAiEIN-E-GW28SpixkfYe47X-7xdHa-Rga1soXFLR8VxuyyK1V3dvtD13q6Fgx9NgOog79YbO_XuFzQ78GoqChMmEgmAubM78IRKy-GfMi4xf06HS9Coh6NOBpazNwrPR3S8I9vBoIZ_eJwy1TQIVn1xJz9kYW5aMRCgb2o0eqwVQpm7xlPvPZFjixqvLLhwPTbfyyMgwtY_o2VW8oJ_9onFRxh6TFLpVchAV6mOtBOyBDHgxpjupaOhlFKoplx7TR-9Pk4o31VdnHERhurRlkum14snPjYiX8qlxyLdMZkTLAEZkOVjee4O2oqNmdzOi4ISWRyhXLbuaLPaWUl0USMld9Xx9kv15ehaSEunTIuol0jQtpwt3Ou9HTqqnxqXIkyv5iAcfK0kDkikZU5SYZHcia4GQcNq52_dk41nRzAAbBQIUI2jyMKPmz8seqMlPJvJ06B0ZLXI76X-kRpe6Kwt2Bj_dlDnCdRUT5VcXBGegzx6QSCr00fGhQ46rWOOuStLtxenIF6rLxHOH4bqpaoJLQEF2IVjRvxCauBTsXsksfmwTBjBavqtAf27LvyV7cfo_m5497r5COTR-WG_vTJ26SRaHJrWS73V3HCnc3dMYKKvou73_wyMmI4tcTrUkAjMXbGilHdaMlSi68v7ldbghwz92N67nXO1s-3umzEW8c5dAiMyrSxQlNCdq4d_HSm538V3yggH_BqkFTz7RgGUlz4XqKzETfI8OUJG5RWOSJXQTGkTTn26RgvIdBXRTBizLAjHi06GpCnXKQDq3umXFhMd5iEf9_Hlq0cOBtd1EtfTMCN0S2ecEX_RxRzD3f3R6_zhuyHLQdgIWWGwlvB0ihn0hqttGPTxN5tUYLujPxWifpYXjD_arcWYC9NdPQmr7eAyQ9vRFba5wpmFFNYHKqOcOa7DR7OfyrjSWJXflxbB0OQ5Aso4z4T4Yj-YzbmRIxx_5-2ok1-YZjs2cDQJvQij7DAKV5LzztFPcHG-lp05RsOX12C80_6wVZ-jTSGZHrjcXK0QWg030sEusqZkBJRv1aizoqWeVtellu9dXt1A6TUnH0RsKiLbD1BMnRwLKperCXaYDwntBtu9LYWWQSBEAl4T9X4xxrJWlW6PyZzobwvS4qLpDPxq6LUIDWaBuJKFqC5i0hdv77X4QQsaEaYMSsfoVlNojIBUruh5uKmLEcmaXKIS6MLpipKr4dml5Pn4o1OTrsWERv5ReD09YVlYYzrSo1PYmBLzlUrF7j8vbPkusVF1RKyrRK4C6CZ5yIYSJZxWVXN0krEbyjiQo1qsfRfJmHACe5BMZzd4kFYEVxVyFegAmMzK__RpvLAj_g6HOM85q3CBymlJ_kEx6GD7fSfKTdw_B308-UEnIywLpa0CfYkfiFNQcCxhf1Jl53Tn8K8r3KNZOPi57Qm9G2uWDPZ_iATErpdLHcfACkNiQlxL1Q64deTaQD3K38hP4opghp1CnyxvVwb4FEuGi7QZYTXbTOrR-c5KLJAqmsxEuTbySFQfk1zMOIB0LRX4KJ6uOc4ygr_p_VttMXE6jZVj1dAbXH0eiySOvi61iG2JzBh8zFO3blP0g7tPw137yCRPqvvnmd2Lqj_JZTxMCgeO3ZjJhPPsREdviCzJXJ-MQuHOLEKBz128Xtr-Li7Hu6yySOXZfdZsC2pLs_UwFD_QUjoI8dQu1ufq5EjnhbjBIfOtn6jFoI5Tv4QEnGb6q2RyjJtmoHtjW9FBq-3zCdf-uZK8HgehoCw-JMvLnqZvmofBfHwpnsx-4glY33_JZMpOBlLdRlDb1mPAuSk92UjZM5PxgcStSBknbdqNy-OG2_XbfwN1ARKhpYoBwjdk2GIM1nm_ynn2rG8h_Pha1zlsU_KD7C5xenT1J3u6QN5k8y4KQs-8NIwfM4XaBDAfjMgWXoTGE7Hijp-KTmUFv0srOYUbLAC7_C5AWbRA1dIsd8O09QX7z5F_KVgQrHeQhvK_qQKkLbmjxjkAmlS82t21Ue75pNWwA5Ok1UCt2IXvQJWX-lfWprUDT73f36M8MsQT_RVkmICm0Oh0tOSbZDX5F5Cc401ilgsevof6jkyINCQpuTKzpyXiFE1ZJUz9QClJU2alGy_egz81irM9pzfjHYBGlVfhKbs132dqJeiMyT-cK6DrUZQ4Zl0PEka1Ngip7J8cAuFtIAE5Ii-e3PssAkxoQjJbpG-h-zWhuw7FgQPLXVrluOker2CBRYQoFsoyQm5r9ALwGWQnJ51Wg7te_tdZ4F2U79lP525QdmYqSmjstdJ--jEnmHwfziZO30uUoRH35eLMXdPpME5qA26LoVKJCQOflRUqlx5aXKzbyBKEOQPudoW6OE_DPorWuGdIZ2R4P0B7sS5pkaMexAWYGyGO0BaIbvn6uO4ljs0PqkjoCz4RkILTMTbCKUlKAhX05D3yaxNO7P_DIKTwsQrPXF_EiVyKs67IFlj7TLKW70JX5EyZqmJxH1PKqZM02bJwT8Rq2ktkTzk6TmciG6bCk7xLeDGqlgBRyIcmZtEYJdKaEqe_VzsYtHxjQAqejN9r5LVI4kHbdqe4NvWfZC662q--Iko8JvrEkhmZD7pT3YdGiKfiax_ckYmPSEq0uBDv_Ph9TLGp3RxnLgrY2KmTMLptP7ZBlXMPUkOSdheNH3YF09kEC0sEpyhM0zL_HVRuDT1l8ZjWhzapAjOWQwZJM2XpYOtGqr7uRQCipUtkB22lqRNSEYHkSwS50iY8shv09dFxkqOjqpUfSF_txDjbTJIJFwKTuLsijnmN3QW677B5KjgSSMNEPbWJJ02Qpb6QV4hUu3U8ORL6XhN9GFsSVfiYDAUBEMCkJSaZ-aBi3Bq41WGjYfr-LvR9QiAL95PHUamHyqi6R3S2fMijfuo5Fgu8J877UHqCODaehReBTBdLMuupqPq6Dz84RZnMLLk2mhKpup8iQBUz69TwMzf9UiU5INo40XwEhAE-5sLSZ7uVp2QDUWchXemzQrR9w_-MexJR-wn6OH1jdesgrM8L719lcLgdttTM6XT8yfX-R5geIxRt0z8sHiVGJVTvUPnZqOdX0J_lNIgmWT4Kxa9ysnNT2RK1ajrDL-nGII2ansJUth4dFkIubQ54V42FjWoBrY4nOKjgYQt0ZNrL9HGHM6YTM9ne1PJjBVn2fYcJytiAy3lP2AIHW6VnMDrLNTTIoqZOyxYBhI0XMaMzEGtNczRzZNe-u-STLiUkUWSDlgQXNdcHRYLYNpAeMkUSSDzRLBkH810iUER9TBRrnTk2Vu4GwY16GrhKYTlRhr0u6NxYZOoC7BQ2qN0BeUcIOo6hxm0A6SJMkTj8t6tqOkTePKRN-QRm6G0YaOQm4w7QB2F9SV7DAQAHN3nzHXfwW5IfVCVKmetaNg9-fUNWdZTNDESO0SG_uKK2z8kntsqiR9p_j1uN9Ke8JLBGUZDfyNJUH4F6BHbBjRH5Yp9NjJhIZZTljqbr6Ms-fXyqA0k7I48VyHMeysRvHwbgzT5PDo4Typ4ARATb3yoMdm-YCG4ff2g4rTQA5NBIlMJiv11O-1-8ehEAS3HVeFBZBLVsprHMoAXkMYZe6fKm9QMlKM_P3L-0yKuUqDMhCm3Xtfozp4aTzQy3N28XxXeFCLMTYHdKQKnI5LwzhEjmXwJcP1P_j2YaDaVry0OJHZ8BXYxtLCptGKyFsN2txCv71GdHjIQsz5EoLm8IhrJL40KK3fWDIfwLfzYjMxEobk3wsNmoBcyydCTlKQ05mvkSUVZm4MKj62uf_F8O-2ukc2xfVVZYZ2cr6bloimFRuBlVKj2D5YKvvKA5s6FEb2-OKjUxYGLbcZ5p5fHLorjj1mbSFZt5srooIwZiOhOQFLOneySZSxpBpREYx8Qk1UvYCErGBjowRVoinVGruLkNmc1ss4SF6ATcJNW1HAdvCq9L_rAzvPXabDPyGEtekj77USHYFWlixJ5Qgl4UgsJGCmEmdJBVF7qlKUE0VJpyjtbMNoURPDvxmBjwZao7cGeERMQZ5UqhkJmA86ndE\",\"content\":[]},{\"id\":\"msg_0a7d8cec3288e849006a84049ac1fc8199aafa166b4289d0ae\",\"content\":[{\"annotations\":[],\"text\":\"The “jumps” between successive terms are \\n 6–2=4, 12–6=6, 20–12=8, 30–20=10,… \\ni.e. you keep adding the even numbers 4,6,8,10,… (which themselves go up by 2 each time). \\n\\nEquivalently, the k-th term is the product of two consecutive integers: \\n 2 = 1×2, \\n 6 = 2×3, \\n12 = 3×4, \\n20 = 4×5, \\n30 = 5×6, … \\n\\nIf we index so that a₁=2, a₂=6, …, then \\n aₙ = n·(n+1) = n² + n.\",\"type\":\"output_text\",\"logprobs\":[]}],\"role\":\"assistant\",\"status\":\"completed\",\"type\":\"message\"},{\"role\":\"user\",\"content\":\"Using the pattern you discovered, what would be the 10th term? And can you find the sum of the first 10 terms?\"}],\"model\":\"o4-mini\",\"reasoning\":{\"effort\":\"high\",\"summary\":\"detailed\"}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-f0262cdce49b.json", + "headers" : { + "x-request-id" : "req_d1f6e34b2da343c291ecab35a56b2634", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a2cf146c5bf47577-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999547", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Tue, 18 Aug 2026 07:07:19 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=41VxFxFHQ6o8QZ9NjWj_mW.EgAkF4BVAGzwIU6_lzZM-1787036827.5784333-1.0.1.1-vBtQ52sMZ8HFfrO4YWTlPEdQCLfxEaeyb2M21a6xgsMafYFPDS7Usah4Nxq7cBL7zqyJtZhiGP7QzYDooV1JB7GX_YMWNGv5cmNo.0s11cbK7GrKorRY0dEQxVhDkC4M; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 18 Aug 2026 07:37:19 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "12076", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "72ab105f-6109-37f6-9e73-8e7837ba889d", + "persistent" : true, + "insertionIndex" : 41 +} \ No newline at end of file