more instrumentation for openai responses api

braintrust-sdk/instrumentation/anthropic_2_2_0/src/main/java/dev/braintrust/instrumentation/anthropic/v2_2_0/TracingHttpClient.java

@@ -24,6 +24,7 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
@@ -51,10 +52,11 @@ public TracingHttpClient(OpenTelemetry openTelemetry, HttpClient underlying) {
      * instrumented — a long-lived client wrapped inside some unrelated span would otherwise parent
      * every future request to that stale span.
      */
-    private Span startLlmSpan(@Nullable Context headerContext) {
+    private Span startLlmSpan(@Nullable Context headerContext, Instant startTime) {
         Context parent = headerContext != null ? headerContext : Context.current();
         return tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME)
                 .setParent(parent)
+                .setStartTimestamp(startTime)
                 .startSpan();
     }
 
@@ -110,7 +112,8 @@ public void close() {
     public @Nonnull HttpResponse execute(
             @Nonnull HttpRequest httpRequest, @Nonnull RequestOptions requestOptions) {
         var extracted = extractCallerContext(httpRequest);
-        var span = startLlmSpan(extracted.callerContext());
+        var llmSpanStart = Instant.now();
+        var span = startLlmSpan(extracted.callerContext(), llmSpanStart);
         try (var ignored = span.makeCurrent()) {
             var bufferedRequest = bufferRequestBody(extracted.request());
 
@@ -128,7 +131,7 @@ public void close() {
                     inputJson);
 
             var response = underlying.execute(bufferedRequest, requestOptions);
-            return new TeeingStreamHttpResponse(response, span);
+            return new TeeingStreamHttpResponse(response, span, tracer, llmSpanStart);
         } catch (Exception e) {
             InstrumentationSemConv.tagLLMSpanResponse(span, e);
             span.end();
@@ -140,7 +143,8 @@ public void close() {
     public @Nonnull CompletableFuture<HttpResponse> executeAsync(
             @Nonnull HttpRequest httpRequest, @Nonnull RequestOptions requestOptions) {
         var extracted = extractCallerContext(httpRequest);
-        var span = startLlmSpan(extracted.callerContext());
+        var llmSpanStart = Instant.now();
+        var span = startLlmSpan(extracted.callerContext(), llmSpanStart);
         try {
             var bufferedRequest = bufferRequestBody(extracted.request());
             String inputJson =
@@ -157,7 +161,10 @@ public void close() {
             return underlying
                     .executeAsync(bufferedRequest, requestOptions)
                     .thenApply(
-                            response -> (HttpResponse) new TeeingStreamHttpResponse(response, span))
+                            response ->
+                                    (HttpResponse)
+                                            new TeeingStreamHttpResponse(
+                                                    response, span, tracer, llmSpanStart))
                     .whenComplete(
                             (response, t) -> {
                                 if (t != null) {
@@ -237,14 +244,19 @@ private static String readBodyAsString(HttpRequestBody body) {
     private static final class TeeingStreamHttpResponse implements HttpResponse {
         private final HttpResponse delegate;
         private final Span span;
+        private final Tracer tracer;
+        private final Instant llmSpanStart;
         private final long spanStartNanos = System.nanoTime();
         private final AtomicLong timeToFirstTokenNanos = new AtomicLong();
         private final ByteArrayOutputStream teeBuffer = new ByteArrayOutputStream();
         private final InputStream teeStream;
 
-        TeeingStreamHttpResponse(HttpResponse delegate, Span span) {
+        TeeingStreamHttpResponse(
+                HttpResponse delegate, Span span, Tracer tracer, Instant llmSpanStart) {
             this.delegate = delegate;
             this.span = span;
+            this.tracer = tracer;
+            this.llmSpanStart = llmSpanStart;
             this.teeStream =
                     new TeeInputStream(
                             delegate.body(), teeBuffer, this::onFirstByte, this::onStreamClosed);
@@ -260,7 +272,18 @@ private void onStreamClosed() {
                 synchronized (teeBuffer) {
                     bytes = teeBuffer.toByteArray();
                 }
-                tagSpanFromBuffer(span, bytes, timeToFirstTokenNanos.get());
+                String responseJson = tagSpanFromBuffer(span, bytes, timeToFirstTokenNanos.get());
+                if (responseJson != null) {
+                    // Emit child spans for server-side tool calls (web search, etc.) nested under
+                    // the LLM span, while it is still live. Anchored at the LLM span start with
+                    // zero duration — providers don't report per-tool timing.
+                    InstrumentationSemConv.addServerSideChildSpans(
+                            tracer,
+                            span,
+                            InstrumentationSemConv.PROVIDER_NAME_ANTHROPIC,
+                            responseJson,
+                            llmSpanStart);
+                }
             } finally {
                 span.end();
             }
@@ -354,8 +377,8 @@ private void notifyClosed() {
     // Span tagging from buffered bytes
     // -------------------------------------------------------------------------
 
-    private static void tagSpanFromBuffer(Span span, byte[] bytes, Long timeToFirstTokenNanos) {
-        if (bytes.length == 0) return;
+    private static String tagSpanFromBuffer(Span span, byte[] bytes, Long timeToFirstTokenNanos) {
+        if (bytes.length == 0) return null;
         try {
             String firstLine = firstNonEmptyLine(bytes);
             // Anthropic SSE starts with "event: message_start\ndata: ..." so we detect
@@ -364,17 +387,17 @@ private static void tagSpanFromBuffer(Span span, byte[] bytes, Long timeToFirstT
                     firstLine != null
                             && (firstLine.startsWith("data:") || firstLine.startsWith("event:"));
             if (isSse) {
-                tagSpanFromSseBytes(span, bytes, timeToFirstTokenNanos);
+                return tagSpanFromSseBytes(span, bytes, timeToFirstTokenNanos);
             } else {
                 // Non-streaming: plain Message JSON — pass it whole, no time_to_first_token
+                String responseJson = new String(bytes, StandardCharsets.UTF_8);
                 InstrumentationSemConv.tagLLMSpanResponse(
-                        span,
-                        InstrumentationSemConv.PROVIDER_NAME_ANTHROPIC,
-                        new String(bytes, StandardCharsets.UTF_8),
-                        null);
+                        span, InstrumentationSemConv.PROVIDER_NAME_ANTHROPIC, responseJson, null);
+                return responseJson;
             }
         } catch (Exception e) {
             log.error("Could not tag span from Anthropic response buffer", e);
+            return null;
         }
     }
 
@@ -405,7 +428,7 @@ private static String firstNonEmptyLine(byte[] bytes) {
      * field inside the JSON. Feed each data payload to {@link MessageAccumulator} and serialize the
      * assembled {@link com.anthropic.models.messages.Message} for the span.
      */
-    private static void tagSpanFromSseBytes(
+    private static String tagSpanFromSseBytes(
             Span span, byte[] sseBytes, Long timeToFirstTokenNanos) {
         try {
             var mapper = BraintrustJsonMapper.get();
@@ -431,8 +454,10 @@ private static void tagSpanFromSseBytes(
                     InstrumentationSemConv.PROVIDER_NAME_ANTHROPIC,
                     assembledMessageJson,
                     timeToFirstTokenNanos);
+            return assembledMessageJson;
         } catch (Exception e) {
             log.error("Could not parse Anthropic SSE buffer to tag streaming span output", e);
+            return null;
         }
     }
 }

braintrust-sdk/instrumentation/anthropic_2_2_0/src/test/java/dev/braintrust/instrumentation/anthropic/v2_2_0/BraintrustAnthropicWebSearchTest.java

@@ -0,0 +1,135 @@
+package dev.braintrust.instrumentation.anthropic.v2_2_0;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import com.anthropic.client.AnthropicClient;
+import com.anthropic.client.okhttp.AnthropicOkHttpClient;
+import com.anthropic.models.messages.MessageCreateParams;
+import com.anthropic.models.messages.Model;
+import com.anthropic.models.messages.WebSearchTool20250305;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import dev.braintrust.TestHarness;
+import dev.braintrust.instrumentation.Instrumenter;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import java.util.List;
+import lombok.SneakyThrows;
+import net.bytebuddy.agent.ByteBuddyAgent;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Verifies that Anthropic server-side web search is captured both as a cost metric on the LLM span
+ * ({@code server_tool_use_web_search_requests}) and as a child {@code type:"tool"} span parented to
+ * the LLM span, giving web search its own cost/latency visibility on the trace timeline.
+ */
+public class BraintrustAnthropicWebSearchTest {
+    private static final String TEST_MODEL = "claude-sonnet-4-5-20250929";
+    private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
+    private static final AttributeKey<String> SPAN_ATTRIBUTES =
+            AttributeKey.stringKey("braintrust.span_attributes");
+    private static final AttributeKey<String> METADATA =
+            AttributeKey.stringKey("braintrust.metadata");
+    private static final AttributeKey<String> METRICS =
+            AttributeKey.stringKey("braintrust.metrics");
+
+    @BeforeAll
+    public static void beforeAll() {
+        var instrumentation = ByteBuddyAgent.install();
+        Instrumenter.install(
+                instrumentation, BraintrustAnthropicWebSearchTest.class.getClassLoader());
+    }
+
+    private TestHarness testHarness;
+
+    @BeforeEach
+    void beforeEach() {
+        testHarness = TestHarness.setup();
+    }
+
+    private static MessageCreateParams webSearchRequest() {
+        return MessageCreateParams.builder()
+                .model(Model.of(TEST_MODEL))
+                .maxTokens(1024)
+                .addUserMessage(
+                        "Search the web for one recent AI news headline and answer in one"
+                                + " sentence.")
+                .addTool(WebSearchTool20250305.builder().maxUses(3).build())
+                .build();
+    }
+
+    @Test
+    @SneakyThrows
+    void testWebSearch() {
+        AnthropicClient client =
+                AnthropicOkHttpClient.builder()
+                        .baseUrl(testHarness.anthropicBaseUrl())
+                        .apiKey(testHarness.anthropicApiKey())
+                        .build();
+
+        var response = client.messages().create(webSearchRequest());
+        assertNotNull(response);
+
+        assertWebSearch(testHarness.awaitExportedSpans(2));
+    }
+
+    @Test
+    @SneakyThrows
+    void testWebSearchStreaming() {
+        AnthropicClient client =
+                AnthropicOkHttpClient.builder()
+                        .baseUrl(testHarness.anthropicBaseUrl())
+                        .apiKey(testHarness.anthropicApiKey())
+                        .build();
+
+        try (var stream = client.messages().createStreaming(webSearchRequest())) {
+            stream.stream().forEach(event -> {});
+        }
+
+        assertWebSearch(testHarness.awaitExportedSpans(2));
+    }
+
+    @SneakyThrows
+    private static void assertWebSearch(List<SpanData> spans) {
+        var llmSpans = spans.stream().filter(s -> isType(s, "llm")).toList();
+        assertEquals(1, llmSpans.size(), "expected a single LLM span");
+        var llm = llmSpans.get(0);
+
+        // Cost metric on the LLM span.
+        JsonNode metrics = JSON_MAPPER.readTree(llm.getAttributes().get(METRICS));
+        assertTrue(
+                metrics.has("server_tool_use_web_search_requests"),
+                "expected server_tool_use_web_search_requests metric, got: " + metrics);
+        assertTrue(metrics.get("server_tool_use_web_search_requests").asDouble() >= 1.0);
+
+        // At least one web_search tool span, parented to the LLM span.
+        var toolSpans =
+                spans.stream()
+                        .filter(s -> isType(s, "tool"))
+                        .filter(s -> "web_search".equals(s.getName()))
+                        .toList();
+        assertFalse(
+                toolSpans.isEmpty(),
+                "expected at least one web_search tool span, got: "
+                        + spans.stream().map(SpanData::getName).toList());
+
+        for (var tool : toolSpans) {
+            assertEquals(
+                    llm.getSpanId(),
+                    tool.getParentSpanId(),
+                    "web_search tool span must be a child of the LLM span");
+            JsonNode metadata = JSON_MAPPER.readTree(tool.getAttributes().get(METADATA));
+            assertEquals("server_tool_use", metadata.path("tool_call_type").asText());
+            assertEquals("web_search_tool_result", metadata.path("tool_result_type").asText());
+            assertFalse(metadata.path("tool_use_id").asText().isEmpty());
+        }
+    }
+
+    @SneakyThrows
+    private static boolean isType(SpanData span, String type) {
+        String attr = span.getAttributes().get(SPAN_ATTRIBUTES);
+        return attr != null && type.equals(JSON_MAPPER.readTree(attr).path("type").asText());
+    }
+}

braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java

@@ -55,7 +55,7 @@ private void tagSpan(
             @Nullable String responseBody) {
         try {
             Map<String, Object> metadata = new java.util.HashMap<>();
-            metadata.put("provider", "gemini");
+            metadata.put("provider", "google");
 
             // Parse request
             if (requestBody != null) {

braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/dev/braintrust/instrumentation/genai/v1_18_0/BraintrustGenAITest.java

@@ -67,7 +67,7 @@ void testWrapGemini() {
                 span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata"));
         assertNotNull(metadataJson, "braintrust.metadata should be set");
         var metadata = JSON_MAPPER.readTree(metadataJson);
-        assertEquals("gemini", metadata.get("provider").asText());
+        assertEquals("google", metadata.get("provider").asText());
         assertEquals(MODEL_ID, metadata.get("model").asText());
         assertEquals(0.0, metadata.get("temperature").asDouble());
         assertEquals(50, metadata.get("maxOutputTokens").asInt());
@@ -145,7 +145,7 @@ void testWrapGeminiAsync() {
                 span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata"));
         assertNotNull(metadataJson, "braintrust.metadata should be set");
         var metadata = JSON_MAPPER.readTree(metadataJson);
-        assertEquals("gemini", metadata.get("provider").asText());
+        assertEquals("google", metadata.get("provider").asText());
         assertEquals(MODEL_ID, metadata.get("model").asText());
         assertEquals(0.0, metadata.get("temperature").asDouble());
         assertEquals(50, metadata.get("maxOutputTokens").asInt());

braintrust-sdk/instrumentation/langchain_1_14_0/build.gradle

@@ -0,0 +1,69 @@
+// Java plugin, toolchain (Java 17 / Adoptium), options.release, and repositories
+// are inherited from the parent's subprojects {} block.
+
+// Minimum langchain4j version that ships the OpenAI Responses API
+// (OpenAiResponsesChatModel / OpenAiResponsesStreamingChatModel, first released in 1.14.0).
+def langchainVersion = '1.14.0'
+// Test against a recent release to exercise forward compatibility (and match the version
+// used to record btx cassettes).
+def langchainTestVersion = '1.19.0'
+
+muzzle {
+  pass {
+    group = 'dev.langchain4j'
+    module = 'langchain4j'
+    versions = "[${langchainVersion},)"
+    extraDependency 'dev.langchain4j:langchain4j-http-client'
+    extraDependency 'dev.langchain4j:langchain4j-open-ai'
+    extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
+    extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8'
+  }
+  // The Responses API classes this module targets did not exist before 1.14.0, so it must
+  // not apply to older releases (langchain_1_8_0 covers [1.8.0,1.14.0)).
+  fail {
+    group = 'dev.langchain4j'
+    module = 'langchain4j'
+    pinVersions '1.13.0'
+    extraDependency 'dev.langchain4j:langchain4j-http-client'
+    extraDependency 'dev.langchain4j:langchain4j-open-ai'
+    extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
+    extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8'
+  }
+}
+
+dependencies {
+  compileOnly project(':braintrust-java-agent:instrumenter')
+  implementation "io.opentelemetry:opentelemetry-api:${otelVersion}"
+  implementation 'com.google.code.findbugs:jsr305:3.0.2' // for @Nullable annotations
+  implementation "org.slf4j:slf4j-api:${slf4jVersion}"
+  implementation project(':braintrust-sdk')
+
+  // ByteBuddy for ElementMatcher types used in instrumentation definitions
+  compileOnly 'net.bytebuddy:byte-buddy:1.17.5'
+
+  // Target libraries — compileOnly because they will be on the app classpath at runtime
+  compileOnly "dev.langchain4j:langchain4j:${langchainVersion}"
+  compileOnly "dev.langchain4j:langchain4j-http-client:${langchainVersion}"
+  compileOnly "dev.langchain4j:langchain4j-open-ai:${langchainVersion}"
+
+  // Test dependencies
+  testImplementation(testFixtures(project(":test-harness")))
+  testImplementation project(':braintrust-java-agent:instrumenter')
+  testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}"
+  testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
+  testImplementation 'net.bytebuddy:byte-buddy-agent:1.17.5'
+  testRuntimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}"
+  testImplementation "dev.langchain4j:langchain4j:${langchainTestVersion}"
+  testImplementation "dev.langchain4j:langchain4j-http-client:${langchainTestVersion}"
+  testImplementation "dev.langchain4j:langchain4j-open-ai:${langchainTestVersion}"
+}
+
+test {
+  useJUnitPlatform()
+  workingDir = rootProject.projectDir
+  testLogging {
+    events "passed", "skipped", "failed"
+    showStandardStreams = true
+    exceptionFormat "full"
+  }
+}