From 05afca4c31b9e0527303c931cd4ebfacf8cd0f0e Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 19 Jun 2026 14:34:04 +0200 Subject: [PATCH 1/4] feat: Add support for `@aws-sdk/client-bedrock-runtime` --- AGENTS.md | 4 + e2e/config/pr-comment-scenarios.json | 9 + e2e/helpers/scenario-harness.ts | 9 + ...bedrock-runtime-v3-1048-auto.cassette.json | 241 +++++++ ...rock-runtime-v3-1048-wrapped.cassette.json | 241 +++++++ ...edrock-runtime-v3-1048-auto.span-tree.json | 236 +++++++ ...bedrock-runtime-v3-1048-auto.span-tree.txt | 196 ++++++ ...ock-runtime-v3-1048-wrapped.span-tree.json | 236 +++++++ ...rock-runtime-v3-1048-wrapped.span-tree.txt | 196 ++++++ .../assertions.ts | 143 ++++ .../cassette-filter.mjs | 42 ++ .../constants.mjs | 6 + .../package.json | 15 + .../pnpm-lock.yaml | 486 ++++++++++++++ .../scenario.impl.mjs | 152 +++++ .../scenario.mjs | 21 + .../scenario.test.ts | 56 ++ .../scenario.ts | 21 + ...coding-agent-v079-auto-hook.span-tree.json | 2 +- ...-coding-agent-v079-auto-hook.span-tree.txt | 2 +- ...i-coding-agent-v079-wrapped.span-tree.json | 2 +- ...pi-coding-agent-v079-wrapped.span-tree.txt | 2 +- js/src/auto-instrumentations/configs/all.ts | 5 + .../configs/bedrock-runtime.test.ts | 88 +++ .../configs/bedrock-runtime.ts | 60 ++ js/src/auto-instrumentations/index.ts | 1 + js/src/exports.ts | 1 + js/src/instrumentation/braintrust-plugin.ts | 16 + js/src/instrumentation/config.ts | 12 + .../core/channel-tracing-utils.ts | 23 +- .../plugins/bedrock-runtime-channels.ts | 34 + .../plugins/bedrock-runtime-common.ts | 81 +++ .../plugins/bedrock-runtime-plugin.test.ts | 209 ++++++ .../plugins/bedrock-runtime-plugin.ts | 621 ++++++++++++++++++ js/src/vendor-sdk-types/bedrock-runtime.ts | 172 +++++ js/src/wrappers/bedrock-runtime.test.ts | 240 +++++++ js/src/wrappers/bedrock-runtime.ts | 110 ++++ 37 files changed, 3985 insertions(+), 6 deletions(-) create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-1048-auto.cassette.json create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-1048-wrapped.cassette.json create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-auto.span-tree.json create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-auto.span-tree.txt create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-wrapped.span-tree.json create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-wrapped.span-tree.txt create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/assertions.ts create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/cassette-filter.mjs create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/constants.mjs create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/package.json create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/pnpm-lock.yaml create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/scenario.impl.mjs create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/scenario.mjs create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/scenario.test.ts create mode 100644 e2e/scenarios/bedrock-runtime-instrumentation/scenario.ts create mode 100644 js/src/auto-instrumentations/configs/bedrock-runtime.test.ts create mode 100644 js/src/auto-instrumentations/configs/bedrock-runtime.ts create mode 100644 js/src/instrumentation/plugins/bedrock-runtime-channels.ts create mode 100644 js/src/instrumentation/plugins/bedrock-runtime-common.ts create mode 100644 js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts create mode 100644 js/src/instrumentation/plugins/bedrock-runtime-plugin.ts create mode 100644 js/src/vendor-sdk-types/bedrock-runtime.ts create mode 100644 js/src/wrappers/bedrock-runtime.test.ts create mode 100644 js/src/wrappers/bedrock-runtime.ts diff --git a/AGENTS.md b/AGENTS.md index c9e368658..1df162486 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,10 @@ mise install # Install toolchain and dependencies pnpm run build # Build all workspace packages (from repo root) ``` +## Instrumentation + +Use the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch. + ## Testing Uses Vitest. Prefer running the **narrowest relevant test** rather than the full suite. diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index ee98b2fba..661e847ba 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -90,6 +90,15 @@ { "variantKey": "groq-v1-auto", "label": "Auto-hook" } ] }, + { + "scenarioDirName": "bedrock-runtime-instrumentation", + "label": "Bedrock Runtime Instrumentation", + "metadataScenario": "bedrock-runtime-instrumentation", + "variants": [ + { "variantKey": "bedrock-runtime-v3-1048-wrapped", "label": "Wrapped" }, + { "variantKey": "bedrock-runtime-v3-1048-auto", "label": "Auto-hook" } + ] + }, { "scenarioDirName": "huggingface-instrumentation", "label": "HuggingFace Instrumentation", diff --git a/e2e/helpers/scenario-harness.ts b/e2e/helpers/scenario-harness.ts index 3ebf61dc8..9455044fe 100644 --- a/e2e/helpers/scenario-harness.ts +++ b/e2e/helpers/scenario-harness.ts @@ -284,6 +284,10 @@ interface ActiveCassetteWiring extends CassetteWiring { const CASSETTE_SERVER_ROUTES: CassetteServerRoute[] = [ { prefix: "/anthropic", upstreamOrigin: "https://api.anthropic.com" }, + { + prefix: "/aws-bedrock-runtime", + upstreamOrigin: "https://bedrock-runtime.us-east-1.amazonaws.com", + }, { prefix: "/cohere", upstreamOrigin: "https://api.cohere.com" }, { prefix: "/cursor/v1", upstreamOrigin: "https://api.cursor.com/v1" }, { prefix: "/cursor", upstreamOrigin: "https://api2.cursor.sh" }, @@ -315,6 +319,7 @@ function getCassetteEnv(wiring: ActiveCassetteWiring): Record { BRAINTRUST_E2E_CASSETTE_VARIANT: wiring.variantKey, BRAINTRUST_E2E_MODEL_BASE_URL: `${serverUrl}/openai/v1`, ANTHROPIC_BASE_URL: `${serverUrl}/anthropic`, + AWS_BEDROCK_RUNTIME_BASE_URL: `${serverUrl}/aws-bedrock-runtime`, COHERE_BASE_URL: `${serverUrl}/cohere`, COHERE_API_URL: `${serverUrl}/cohere`, CURSOR_BACKEND_URL: `${serverUrl}/cursor`, @@ -362,6 +367,10 @@ const CASSETTE_PROVIDER_KEYS: Array<{ envVars: ["ANTHROPIC_API_KEY"], placeholder: "sk-ant-cassette-placeholder", }, + { + envVars: ["AWS_BEARER_TOKEN_BEDROCK"], + placeholder: "btb-cassette-placeholder", + }, { envVars: ["COHERE_API_KEY", "CO_API_KEY"], placeholder: "cassette-placeholder", diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-1048-auto.cassette.json b/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-1048-auto.cassette.json new file mode 100644 index 000000000..b030ca707 --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-1048-auto.cassette.json @@ -0,0 +1,241 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "322a4f0839625dc9", + "matchKey": "POST bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse", + "recordedAt": "2026-06-18T14:58:39.662Z", + "request": { + "body": { + "kind": "json", + "value": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly OK." + } + ], + "role": "user" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse" + }, + "response": { + "body": { + "kind": "json", + "value": { + "metrics": { + "latencyMs": 470 + }, + "output": { + "message": { + "content": [ + { + "text": "OK." + } + ], + "role": "assistant" + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 5, + "outputTokens": 3, + "serverToolUsage": {}, + "totalTokens": 8 + } + } + }, + "headers": { + "connection": "keep-alive", + "content-length": "202", + "content-type": "application/json", + "date": "[REDACTED]", + "x-amzn-requestid": "[REDACTED]" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "d98c8ffcf884de04", + "matchKey": "POST bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse-stream", + "recordedAt": "2026-06-18T14:58:40.359Z", + "request": { + "body": { + "kind": "json", + "value": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly STREAM." + } + ], + "role": "user" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse-stream" + }, + "response": { + "body": { + "contentType": "application/vnd.amazon.eventstream", + "kind": "base64", + "value": "AAAAugAAAFK6MP8ECzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNDU2NzgiLCJyb2xlIjoiYXNzaXN0YW50In00xKBOAAAAsgAAAFf6KkBKCzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IiJ9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eSJ9WxjlpwAAAMwAAABXvMge5As6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiJTVFJFQU0uIn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUiJ9xVBV1gAAALUAAABWPw2szAs6ZXZlbnQtdHlwZQcAEGNvbnRlbnRCbG9ja1N0b3ANOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVlcifY0Un70AAACxAAAAUVTpn68LOmV2ZW50LXR5cGUHAAttZXNzYWdlU3RvcA06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7InAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVYiLCJzdG9wUmVhc29uIjoiZW5kX3R1cm4ifcT0cWcAAADpAAAATtGCFlALOmV2ZW50LXR5cGUHAAhtZXRhZGF0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7Im1ldHJpY3MiOnsibGF0ZW5jeU1zIjo1NDl9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1diIsInVzYWdlIjp7ImlucHV0VG9rZW5zIjo2LCJvdXRwdXRUb2tlbnMiOjQsInNlcnZlclRvb2xVc2FnZSI6e30sInRvdGFsVG9rZW5zIjoxMH19HJaMFw==" + }, + "headers": { + "connection": "keep-alive", + "content-type": "application/vnd.amazon.eventstream", + "date": "[REDACTED]", + "transfer-encoding": "chunked", + "x-amzn-requestid": "[REDACTED]" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "0f167aed4ca1a787", + "matchKey": "POST bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke", + "recordedAt": "2026-06-18T14:58:41.002Z", + "request": { + "body": { + "kind": "json", + "value": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly RAW." + } + ], + "role": "user" + } + ], + "schemaVersion": "messages-v1" + } + }, + "headers": {}, + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke" + }, + "response": { + "body": { + "kind": "json", + "value": { + "output": { + "message": { + "content": [ + { + "text": "RAW" + } + ], + "role": "assistant" + } + }, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 0, + "inputTokens": 5, + "outputTokens": 2, + "totalTokens": 7 + } + } + }, + "headers": { + "connection": "keep-alive", + "content-length": "212", + "content-type": "application/json", + "date": "[REDACTED]", + "x-amzn-bedrock-cache-read-input-token-count": "0", + "x-amzn-bedrock-cache-write-input-token-count": "0", + "x-amzn-bedrock-input-token-count": "5", + "x-amzn-bedrock-invocation-latency": "482", + "x-amzn-bedrock-output-token-count": "2", + "x-amzn-requestid": "[REDACTED]" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "79f562ce7e378bb8", + "matchKey": "POST bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke-with-response-stream", + "recordedAt": "2026-06-18T14:58:41.756Z", + "request": { + "body": { + "kind": "json", + "value": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly RAWSTREAM." + } + ], + "role": "user" + } + ], + "schemaVersion": "messages-v1" + } + }, + "headers": {}, + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke-with-response-stream" + }, + "response": { + "body": { + "contentType": "application/vnd.amazon.eventstream", + "kind": "base64", + "value": "AAAAzAAAAEuoyUKrCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWE56WVdkbFUzUmhjblFpT25zaWNtOXNaU0k2SW1GemMybHpkR0Z1ZENKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QIn0ANHGfAAAA+gAAAEuGqA+NCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRFWld4MFlTSTZleUprWld4MFlTSTZleUowWlhoMElqb2lVMjl5Y25raWZTd2lZMjl1ZEdWdWRFSnNiMk5yU1c1a1pYZ2lPakI5ZlE9PSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUiJ9i9xo5AAAALQAAABLYWvppQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0VGRHOXdJanA3SW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpvd2ZYMD0iLCJwIjoiYWJjZGVmZ2hpaiJ9pf0sLAAAAQoAAABLBcCMdQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0RVpXeDBZU0k2ZXlKa1pXeDBZU0k2ZXlKMFpYaDBJam9pTENCSklHTmhiaWQwSUhCeWIzWnBaR1VpZlN3aVkyOXVkR1Z1ZEVKc2IyTnJTVzVrWlhnaU9qRjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSIn0yQ/kYAAAA0wAAAEtKeUL4CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRUZEc5d0lqcDdJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam94ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OTyJ9UkcPkwAAAOYAAABLI7h1Dgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0RVpXeDBZU0k2ZXlKa1pXeDBZU0k2ZXlKMFpYaDBJam9pSUdGdWVTSjlMQ0pqYjI1MFpXNTBRbXh2WTJ0SmJtUmxlQ0k2TW4xOSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCIn2nV6NWAAAAxwAAAEvfGXO6CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRUZEc5d0lqcDdJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam95ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQyJ9LIy5TwAAAQoAAABLBcCMdQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0RVpXeDBZU0k2ZXlKa1pXeDBZU0k2ZXlKMFpYaDBJam9pSUdsdVptOXliV0YwYVc5dUluMHNJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam96ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaIn1BBNKDAAAAtgAAAEsbq7rFCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRUZEc5d0lqcDdJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam96ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2wifW1x/nYAAAEWAAAAS6DQ9vYLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydEVaV3gwWVNJNmV5SmtaV3gwWVNJNmV5SjBaWGgwSWpvaUlIUm9ZWFFnYldsbmFIUWdZbVVnWTI5dVptbGtaVzUwYVdGc0luMHNJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam8wZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUiJ9Tm4t9gAAAL4AAABLK9vxBAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0VGRHOXdJanA3SW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpvMGZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3QifeKvBBcAAADxAAAAS/F4PpwLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydEVaV3gwWVNJNmV5SmtaV3gwWVNJNmV5SjBaWGgwSWpvaUlHOXlJbjBzSW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpvMWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNIn29j01mAAAA2wAAAEt6CQk5CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRUZEc5d0lqcDdJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam8xZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXIn2sKgwmAAABCgAAAEsFwIx1CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRFWld4MFlTSTZleUprWld4MFlTSTZleUowWlhoMElqb2lJSEJ5YjNCeWFXVjBZWEo1SW4wc0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqbzJmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVoifQSBnrQAAACwAAAAS5TrT2ULOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydFRkRzl3SWpwN0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqbzJmWDA9IiwicCI6ImFiY2RlZiJ9gfl3+gAAANAAAABLDdk4KAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0RVpXeDBZU0k2ZXlKa1pXeDBZU0k2ZXlKMFpYaDBJam9pTGlKOUxDSmpiMjUwWlc1MFFteHZZMnRKYm1SbGVDSTZOMzE5IiwicCI6ImFiY2RlZmdoaWoifV5+lfYAAAC/AAAASxa72LQLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydFRkRzl3SWpwN0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqbzNmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dSJ9FTIxEQAAALQAAABLYWvppQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUp0WlhOellXZGxVM1J2Y0NJNmV5SnpkRzl3VW1WaGMyOXVJam9pYldGNFgzUnZhMlZ1Y3lKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpaiJ9d8gOLgAAAl8AAABLc6e+YAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUp0WlhSaFpHRjBZU0k2ZXlKMWMyRm5aU0k2ZXlKcGJuQjFkRlJ2YTJWdWN5STZOaXdpYjNWMGNIVjBWRzlyWlc1eklqb3hOaXdpWTJGamFHVlNaV0ZrU1c1d2RYUlViMnRsYmtOdmRXNTBJam93TENKallXTm9aVmR5YVhSbFNXNXdkWFJVYjJ0bGJrTnZkVzUwSWpvd2ZTd2liV1YwY21samN5STZlMzBzSW5SeVlXTmxJanA3Zlgwc0ltRnRZWHB2YmkxaVpXUnliMk5yTFdsdWRtOWpZWFJwYjI1TlpYUnlhV056SWpwN0ltbHVjSFYwVkc5clpXNURiM1Z1ZENJNk5pd2liM1YwY0hWMFZHOXJaVzVEYjNWdWRDSTZNVFlzSW1sdWRtOWpZWFJwYjI1TVlYUmxibU41SWpvMk1ERXNJbVpwY25OMFFubDBaVXhoZEdWdVkza2lPalV4TkN3aVkyRmphR1ZTWldGa1NXNXdkWFJVYjJ0bGJrTnZkVzUwSWpvd0xDSmpZV05vWlZkeWFYUmxTVzV3ZFhSVWIydGxia052ZFc1MElqb3dmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNDU2Nzgifck4p6o=" + }, + "headers": { + "connection": "keep-alive", + "content-type": "application/vnd.amazon.eventstream", + "date": "[REDACTED]", + "transfer-encoding": "chunked", + "x-amzn-bedrock-content-type": "application/json", + "x-amzn-requestid": "[REDACTED]" + }, + "status": 200, + "statusText": "OK" + } + } + ], + "meta": { + "createdAt": "2026-06-18T14:56:42.290Z" + } +} diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-1048-wrapped.cassette.json b/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-1048-wrapped.cassette.json new file mode 100644 index 000000000..28ce827b0 --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-1048-wrapped.cassette.json @@ -0,0 +1,241 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "322a4f0839625dc9", + "matchKey": "POST bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse", + "recordedAt": "2026-06-18T14:58:35.988Z", + "request": { + "body": { + "kind": "json", + "value": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly OK." + } + ], + "role": "user" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse" + }, + "response": { + "body": { + "kind": "json", + "value": { + "metrics": { + "latencyMs": 342 + }, + "output": { + "message": { + "content": [ + { + "text": "OK." + } + ], + "role": "assistant" + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 5, + "outputTokens": 3, + "serverToolUsage": {}, + "totalTokens": 8 + } + } + }, + "headers": { + "connection": "keep-alive", + "content-length": "202", + "content-type": "application/json", + "date": "[REDACTED]", + "x-amzn-requestid": "[REDACTED]" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "d98c8ffcf884de04", + "matchKey": "POST bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse-stream", + "recordedAt": "2026-06-18T14:58:36.755Z", + "request": { + "body": { + "kind": "json", + "value": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly STREAM." + } + ], + "role": "user" + } + ] + } + }, + "headers": {}, + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse-stream" + }, + "response": { + "body": { + "contentType": "application/vnd.amazon.eventstream", + "kind": "base64", + "value": "AAAAggAAAFIrYQxDCzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlIiwicm9sZSI6ImFzc2lzdGFudCJ9Ki5zGAAAAMsAAABXDujC9As6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiIifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYIn3BwnKHAAAAwwAAAFc+mIk1CzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IlNUUkVBTS4ifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJIn3cYal2AAAArQAAAFZvnXCPCzpldmVudC10eXBlBwAQY29udGVudEJsb2NrU3RvcA06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk8ifcKfBB4AAACFAAAAUQBIgekLOmV2ZW50LXR5cGUHAAttZXNzYWdlU3RvcA06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7InAiOiJhYmNkIiwic3RvcFJlYXNvbiI6ImVuZF90dXJuIn0c+t0FAAAA3wAAAE7/41t2CzpldmVudC10eXBlBwAIbWV0YWRhdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJtZXRyaWNzIjp7ImxhdGVuY3lNcyI6NjE3fSwicCI6ImFiY2RlZmdoaWprbCIsInVzYWdlIjp7ImlucHV0VG9rZW5zIjo2LCJvdXRwdXRUb2tlbnMiOjQsInNlcnZlclRvb2xVc2FnZSI6e30sInRvdGFsVG9rZW5zIjoxMH19T7Dl9A==" + }, + "headers": { + "connection": "keep-alive", + "content-type": "application/vnd.amazon.eventstream", + "date": "[REDACTED]", + "transfer-encoding": "chunked", + "x-amzn-requestid": "[REDACTED]" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "0f167aed4ca1a787", + "matchKey": "POST bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke", + "recordedAt": "2026-06-18T14:58:37.567Z", + "request": { + "body": { + "kind": "json", + "value": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly RAW." + } + ], + "role": "user" + } + ], + "schemaVersion": "messages-v1" + } + }, + "headers": {}, + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke" + }, + "response": { + "body": { + "kind": "json", + "value": { + "output": { + "message": { + "content": [ + { + "text": "RAW" + } + ], + "role": "assistant" + } + }, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 0, + "inputTokens": 5, + "outputTokens": 2, + "totalTokens": 7 + } + } + }, + "headers": { + "connection": "keep-alive", + "content-length": "212", + "content-type": "application/json", + "date": "[REDACTED]", + "x-amzn-bedrock-cache-read-input-token-count": "0", + "x-amzn-bedrock-cache-write-input-token-count": "0", + "x-amzn-bedrock-input-token-count": "5", + "x-amzn-bedrock-invocation-latency": "665", + "x-amzn-bedrock-output-token-count": "2", + "x-amzn-requestid": "[REDACTED]" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "79f562ce7e378bb8", + "matchKey": "POST bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke-with-response-stream", + "recordedAt": "2026-06-18T14:58:38.329Z", + "request": { + "body": { + "kind": "json", + "value": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly RAWSTREAM." + } + ], + "role": "user" + } + ], + "schemaVersion": "messages-v1" + } + }, + "headers": {}, + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke-with-response-stream" + }, + "response": { + "body": { + "contentType": "application/vnd.amazon.eventstream", + "kind": "base64", + "value": "AAAAvgAAAEsr2/EECzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWE56WVdkbFUzUmhjblFpT25zaWNtOXNaU0k2SW1GemMybHpkR0Z1ZENKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQiJ9NOrvXwAAAOUAAABLZBgP3gs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0RVpXeDBZU0k2ZXlKa1pXeDBZU0k2ZXlKMFpYaDBJam9pVWtGWEluMHNJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam93ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekEifVDFRRUAAACvAAAAS3ZbTzYLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydFRkRzl3SWpwN0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqb3dmWDA9IiwicCI6ImFiY2RlIn0EwbqNAAAA5wAAAEse2Fy+CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRFWld4MFlTSTZleUprWld4MFlTSTZleUowWlhoMElqb2lVMVJTUlVGTkluMHNJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam94ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5In37xV3mAAAAwAAAAEttOa+qCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRUZEc5d0lqcDdJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam94ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2In0n/4LKAAAAxAAAAEuYuQlqCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWE56WVdkbFUzUnZjQ0k2ZXlKemRHOXdVbVZoYzI5dUlqb2laVzVrWDNSMWNtNGlmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDRCJ9luC75QAAAj8AAABL6pXJLQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUp0WlhSaFpHRjBZU0k2ZXlKMWMyRm5aU0k2ZXlKcGJuQjFkRlJ2YTJWdWN5STZOaXdpYjNWMGNIVjBWRzlyWlc1eklqb3pMQ0pqWVdOb1pWSmxZV1JKYm5CMWRGUnZhMlZ1UTI5MWJuUWlPakFzSW1OaFkyaGxWM0pwZEdWSmJuQjFkRlJ2YTJWdVEyOTFiblFpT2pCOUxDSnRaWFJ5YVdOeklqcDdmU3dpZEhKaFkyVWlPbnQ5ZlN3aVlXMWhlbTl1TFdKbFpISnZZMnN0YVc1MmIyTmhkR2x2YmsxbGRISnBZM01pT25zaWFXNXdkWFJVYjJ0bGJrTnZkVzUwSWpvMkxDSnZkWFJ3ZFhSVWIydGxia052ZFc1MElqb3pMQ0pwYm5adlkyRjBhVzl1VEdGMFpXNWplU0k2TlRBeUxDSm1hWEp6ZEVKNWRHVk1ZWFJsYm1ONUlqbzBOamtzSW1OaFkyaGxVbVZoWkVsdWNIVjBWRzlyWlc1RGIzVnVkQ0k2TUN3aVkyRmphR1ZYY21sMFpVbHVjSFYwVkc5clpXNURiM1Z1ZENJNk1IMTkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHIn1cwjZV" + }, + "headers": { + "connection": "keep-alive", + "content-type": "application/vnd.amazon.eventstream", + "date": "[REDACTED]", + "transfer-encoding": "chunked", + "x-amzn-bedrock-content-type": "application/json", + "x-amzn-requestid": "[REDACTED]" + }, + "status": 200, + "statusText": "OK" + } + } + ], + "meta": { + "createdAt": "2026-06-18T14:56:41.138Z" + } +} diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-auto.span-tree.json b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-auto.span-tree.json new file mode 100644 index 000000000..341dbd92c --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-auto.span-tree.json @@ -0,0 +1,236 @@ +{ + "span_tree": [ + { + "name": "bedrock-runtime-instrumentation-root", + "type": "task", + "children": [ + { + "name": "bedrock-converse-operation", + "children": [ + { + "name": "bedrock.converse", + "type": "llm", + "children": [], + "input": { + "messages": [ + { + "content": [ + { + "text": "Reply with exactly OK." + } + ], + "role": "user" + } + ] + }, + "output": { + "content": [ + { + "text": "OK." + } + ], + "role": "assistant" + }, + "metadata": { + "command": "ConverseCommand", + "maxTokens": 16, + "model": "us.amazon.nova-lite-v1:0", + "operation": "converse", + "provider": "aws-bedrock", + "stopReason": "end_turn", + "temperature": 0, + "topP": 0.9 + }, + "metrics": { + "completion_tokens": 3, + "latency_ms": 0, + "prompt_tokens": 5, + "tokens": 8 + } + } + ], + "metadata": { + "operation": "converse", + "testRunId": "" + } + }, + { + "name": "bedrock-converse-stream-operation", + "children": [ + { + "name": "bedrock.converseStream", + "type": "llm", + "children": [], + "input": { + "messages": [ + { + "content": [ + { + "text": "Reply with exactly STREAM." + } + ], + "role": "user" + } + ] + }, + "output": { + "content": [ + { + "text": "STREAM." + } + ], + "role": "assistant" + }, + "metadata": { + "command": "ConverseStreamCommand", + "maxTokens": 16, + "model": "us.amazon.nova-lite-v1:0", + "operation": "converseStream", + "provider": "aws-bedrock", + "stopReason": "end_turn", + "temperature": 0, + "topP": 0.9 + }, + "metrics": { + "completion_tokens": 4, + "latency_ms": 0, + "prompt_tokens": 6, + "time_to_first_token": 0, + "tokens": 10 + } + } + ], + "metadata": { + "operation": "converse-stream", + "testRunId": "" + } + }, + { + "name": "bedrock-invoke-model-operation", + "children": [ + { + "name": "bedrock.invokeModel", + "type": "llm", + "children": [], + "input": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly RAW." + } + ], + "role": "user" + } + ], + "schemaVersion": "messages-v1" + }, + "output": { + "output": { + "message": { + "content": [ + { + "text": "RAW" + } + ], + "role": "assistant" + } + }, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 0, + "inputTokens": 5, + "outputTokens": 2, + "totalTokens": 7 + } + }, + "metadata": { + "accept": "application/json", + "command": "InvokeModelCommand", + "contentType": "application/json", + "model": "us.amazon.nova-lite-v1:0", + "operation": "invokeModel", + "provider": "aws-bedrock" + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 5, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "invoke-model", + "testRunId": "" + } + }, + { + "name": "bedrock-invoke-model-stream-operation", + "children": [ + { + "name": "bedrock.invokeModelWithResponseStream", + "type": "llm", + "children": [], + "input": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly RAWSTREAM." + } + ], + "role": "user" + } + ], + "schemaVersion": "messages-v1" + }, + "output": { + "text": "Sorry, I can't provide any information that might be confidential or proprietary." + }, + "metadata": { + "accept": "application/json", + "command": "InvokeModelWithResponseStreamCommand", + "contentType": "application/json", + "metrics": {}, + "model": "us.amazon.nova-lite-v1:0", + "operation": "invokeModelWithResponseStream", + "provider": "aws-bedrock", + "trace": {}, + "usage": { + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 0, + "inputTokens": 6, + "outputTokens": 16 + } + }, + "metrics": { + "completion_tokens": 16, + "prompt_tokens": 6, + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "invoke-model-stream", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "bedrock-runtime-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-auto.span-tree.txt b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-auto.span-tree.txt new file mode 100644 index 000000000..ba1d8829a --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-auto.span-tree.txt @@ -0,0 +1,196 @@ +span_tree: +└── bedrock-runtime-instrumentation-root [task] + metadata: { + "scenario": "bedrock-runtime-instrumentation", + "testRunId": "" + } + ├── bedrock-converse-operation + │ metadata: { + │ "operation": "converse", + │ "testRunId": "" + │ } + │ └── bedrock.converse [llm] + │ input: { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with exactly OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "text": "OK." + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "command": "ConverseCommand", + │ "maxTokens": 16, + │ "model": "us.amazon.nova-lite-v1:0", + │ "operation": "converse", + │ "provider": "aws-bedrock", + │ "stopReason": "end_turn", + │ "temperature": 0, + │ "topP": 0.9 + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "latency_ms": 0, + │ "prompt_tokens": 5, + │ "tokens": 8 + │ } + ├── bedrock-converse-stream-operation + │ metadata: { + │ "operation": "converse-stream", + │ "testRunId": "" + │ } + │ └── bedrock.converseStream [llm] + │ input: { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with exactly STREAM." + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "text": "STREAM." + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "command": "ConverseStreamCommand", + │ "maxTokens": 16, + │ "model": "us.amazon.nova-lite-v1:0", + │ "operation": "converseStream", + │ "provider": "aws-bedrock", + │ "stopReason": "end_turn", + │ "temperature": 0, + │ "topP": 0.9 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "latency_ms": 0, + │ "prompt_tokens": 6, + │ "time_to_first_token": 0, + │ "tokens": 10 + │ } + ├── bedrock-invoke-model-operation + │ metadata: { + │ "operation": "invoke-model", + │ "testRunId": "" + │ } + │ └── bedrock.invokeModel [llm] + │ input: { + │ "inferenceConfig": { + │ "maxTokens": 16, + │ "temperature": 0, + │ "topP": 0.9 + │ }, + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with exactly RAW." + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "schemaVersion": "messages-v1" + │ } + │ output: { + │ "output": { + │ "message": { + │ "content": [ + │ { + │ "text": "RAW" + │ } + │ ], + │ "role": "assistant" + │ } + │ }, + │ "stopReason": "end_turn", + │ "usage": { + │ "cacheReadInputTokenCount": 0, + │ "cacheWriteInputTokenCount": 0, + │ "inputTokens": 5, + │ "outputTokens": 2, + │ "totalTokens": 7 + │ } + │ } + │ metadata: { + │ "accept": "application/json", + │ "command": "InvokeModelCommand", + │ "contentType": "application/json", + │ "model": "us.amazon.nova-lite-v1:0", + │ "operation": "invokeModel", + │ "provider": "aws-bedrock" + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 5, + │ "tokens": 7 + │ } + └── bedrock-invoke-model-stream-operation + metadata: { + "operation": "invoke-model-stream", + "testRunId": "" + } + └── bedrock.invokeModelWithResponseStream [llm] + input: { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly RAWSTREAM." + } + ], + "role": "user" + } + ], + "schemaVersion": "messages-v1" + } + output: { + "text": "Sorry, I can't provide any information that might be confidential or proprietary." + } + metadata: { + "accept": "application/json", + "command": "InvokeModelWithResponseStreamCommand", + "contentType": "application/json", + "metrics": {}, + "model": "us.amazon.nova-lite-v1:0", + "operation": "invokeModelWithResponseStream", + "provider": "aws-bedrock", + "trace": {}, + "usage": { + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 0, + "inputTokens": 6, + "outputTokens": 16 + } + } + metrics: { + "completion_tokens": 16, + "prompt_tokens": 6, + "time_to_first_token": 0 + } diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-wrapped.span-tree.json b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-wrapped.span-tree.json new file mode 100644 index 000000000..553eeaaa8 --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-wrapped.span-tree.json @@ -0,0 +1,236 @@ +{ + "span_tree": [ + { + "name": "bedrock-runtime-instrumentation-root", + "type": "task", + "children": [ + { + "name": "bedrock-converse-operation", + "children": [ + { + "name": "bedrock.converse", + "type": "llm", + "children": [], + "input": { + "messages": [ + { + "content": [ + { + "text": "Reply with exactly OK." + } + ], + "role": "user" + } + ] + }, + "output": { + "content": [ + { + "text": "OK." + } + ], + "role": "assistant" + }, + "metadata": { + "command": "ConverseCommand", + "maxTokens": 16, + "model": "us.amazon.nova-lite-v1:0", + "operation": "converse", + "provider": "aws-bedrock", + "stopReason": "end_turn", + "temperature": 0, + "topP": 0.9 + }, + "metrics": { + "completion_tokens": 3, + "latency_ms": 0, + "prompt_tokens": 5, + "tokens": 8 + } + } + ], + "metadata": { + "operation": "converse", + "testRunId": "" + } + }, + { + "name": "bedrock-converse-stream-operation", + "children": [ + { + "name": "bedrock.converseStream", + "type": "llm", + "children": [], + "input": { + "messages": [ + { + "content": [ + { + "text": "Reply with exactly STREAM." + } + ], + "role": "user" + } + ] + }, + "output": { + "content": [ + { + "text": "STREAM." + } + ], + "role": "assistant" + }, + "metadata": { + "command": "ConverseStreamCommand", + "maxTokens": 16, + "model": "us.amazon.nova-lite-v1:0", + "operation": "converseStream", + "provider": "aws-bedrock", + "stopReason": "end_turn", + "temperature": 0, + "topP": 0.9 + }, + "metrics": { + "completion_tokens": 4, + "latency_ms": 0, + "prompt_tokens": 6, + "time_to_first_token": 0, + "tokens": 10 + } + } + ], + "metadata": { + "operation": "converse-stream", + "testRunId": "" + } + }, + { + "name": "bedrock-invoke-model-operation", + "children": [ + { + "name": "bedrock.invokeModel", + "type": "llm", + "children": [], + "input": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly RAW." + } + ], + "role": "user" + } + ], + "schemaVersion": "messages-v1" + }, + "output": { + "output": { + "message": { + "content": [ + { + "text": "RAW" + } + ], + "role": "assistant" + } + }, + "stopReason": "end_turn", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 0, + "inputTokens": 5, + "outputTokens": 2, + "totalTokens": 7 + } + }, + "metadata": { + "accept": "application/json", + "command": "InvokeModelCommand", + "contentType": "application/json", + "model": "us.amazon.nova-lite-v1:0", + "operation": "invokeModel", + "provider": "aws-bedrock" + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 5, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "invoke-model", + "testRunId": "" + } + }, + { + "name": "bedrock-invoke-model-stream-operation", + "children": [ + { + "name": "bedrock.invokeModelWithResponseStream", + "type": "llm", + "children": [], + "input": { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly RAWSTREAM." + } + ], + "role": "user" + } + ], + "schemaVersion": "messages-v1" + }, + "output": { + "text": "RAWSTREAM" + }, + "metadata": { + "accept": "application/json", + "command": "InvokeModelWithResponseStreamCommand", + "contentType": "application/json", + "metrics": {}, + "model": "us.amazon.nova-lite-v1:0", + "operation": "invokeModelWithResponseStream", + "provider": "aws-bedrock", + "trace": {}, + "usage": { + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 0, + "inputTokens": 6, + "outputTokens": 3 + } + }, + "metrics": { + "completion_tokens": 3, + "prompt_tokens": 6, + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "invoke-model-stream", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "bedrock-runtime-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-wrapped.span-tree.txt b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-wrapped.span-tree.txt new file mode 100644 index 000000000..d89fb7be8 --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-1048-wrapped.span-tree.txt @@ -0,0 +1,196 @@ +span_tree: +└── bedrock-runtime-instrumentation-root [task] + metadata: { + "scenario": "bedrock-runtime-instrumentation", + "testRunId": "" + } + ├── bedrock-converse-operation + │ metadata: { + │ "operation": "converse", + │ "testRunId": "" + │ } + │ └── bedrock.converse [llm] + │ input: { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with exactly OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "text": "OK." + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "command": "ConverseCommand", + │ "maxTokens": 16, + │ "model": "us.amazon.nova-lite-v1:0", + │ "operation": "converse", + │ "provider": "aws-bedrock", + │ "stopReason": "end_turn", + │ "temperature": 0, + │ "topP": 0.9 + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "latency_ms": 0, + │ "prompt_tokens": 5, + │ "tokens": 8 + │ } + ├── bedrock-converse-stream-operation + │ metadata: { + │ "operation": "converse-stream", + │ "testRunId": "" + │ } + │ └── bedrock.converseStream [llm] + │ input: { + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with exactly STREAM." + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "text": "STREAM." + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "command": "ConverseStreamCommand", + │ "maxTokens": 16, + │ "model": "us.amazon.nova-lite-v1:0", + │ "operation": "converseStream", + │ "provider": "aws-bedrock", + │ "stopReason": "end_turn", + │ "temperature": 0, + │ "topP": 0.9 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "latency_ms": 0, + │ "prompt_tokens": 6, + │ "time_to_first_token": 0, + │ "tokens": 10 + │ } + ├── bedrock-invoke-model-operation + │ metadata: { + │ "operation": "invoke-model", + │ "testRunId": "" + │ } + │ └── bedrock.invokeModel [llm] + │ input: { + │ "inferenceConfig": { + │ "maxTokens": 16, + │ "temperature": 0, + │ "topP": 0.9 + │ }, + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with exactly RAW." + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "schemaVersion": "messages-v1" + │ } + │ output: { + │ "output": { + │ "message": { + │ "content": [ + │ { + │ "text": "RAW" + │ } + │ ], + │ "role": "assistant" + │ } + │ }, + │ "stopReason": "end_turn", + │ "usage": { + │ "cacheReadInputTokenCount": 0, + │ "cacheWriteInputTokenCount": 0, + │ "inputTokens": 5, + │ "outputTokens": 2, + │ "totalTokens": 7 + │ } + │ } + │ metadata: { + │ "accept": "application/json", + │ "command": "InvokeModelCommand", + │ "contentType": "application/json", + │ "model": "us.amazon.nova-lite-v1:0", + │ "operation": "invokeModel", + │ "provider": "aws-bedrock" + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 5, + │ "tokens": 7 + │ } + └── bedrock-invoke-model-stream-operation + metadata: { + "operation": "invoke-model-stream", + "testRunId": "" + } + └── bedrock.invokeModelWithResponseStream [llm] + input: { + "inferenceConfig": { + "maxTokens": 16, + "temperature": 0, + "topP": 0.9 + }, + "messages": [ + { + "content": [ + { + "text": "Reply with exactly RAWSTREAM." + } + ], + "role": "user" + } + ], + "schemaVersion": "messages-v1" + } + output: { + "text": "RAWSTREAM" + } + metadata: { + "accept": "application/json", + "command": "InvokeModelWithResponseStreamCommand", + "contentType": "application/json", + "metrics": {}, + "model": "us.amazon.nova-lite-v1:0", + "operation": "invokeModelWithResponseStream", + "provider": "aws-bedrock", + "trace": {}, + "usage": { + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 0, + "inputTokens": 6, + "outputTokens": 3 + } + } + metrics: { + "completion_tokens": 3, + "prompt_tokens": 6, + "time_to_first_token": 0 + } diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/assertions.ts b/e2e/scenarios/bedrock-runtime-instrumentation/assertions.ts new file mode 100644 index 000000000..f33fcb28c --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/assertions.ts @@ -0,0 +1,143 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; +import { + withScenarioHarness, + type ScenarioRunContext, +} from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; +import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; +import { MODEL, ROOT_NAME, SCENARIO_NAME } from "./constants.mjs"; + +type RunBedrockRuntimeScenario = (harness: { + runNodeScenarioDir: (options: { + entry: string; + nodeArgs: string[]; + runContext?: ScenarioRunContext; + scenarioDir: string; + timeoutMs: number; + }) => Promise; + runScenarioDir: (options: { + entry: string; + runContext?: ScenarioRunContext; + scenarioDir: string; + timeoutMs: number; + }) => Promise; +}) => Promise; + +const OPERATION_TO_SPAN_NAME = { + "bedrock-converse-operation": "bedrock.converse", + "bedrock-converse-stream-operation": "bedrock.converseStream", + "bedrock-invoke-model-operation": "bedrock.invokeModel", + "bedrock-invoke-model-stream-operation": + "bedrock.invokeModelWithResponseStream", +}; + +function findBedrockSpan( + events: CapturedLogEvent[], + operationName: keyof typeof OPERATION_TO_SPAN_NAME, +) { + const operation = findLatestSpan(events, operationName); + const spans = findChildSpans( + events, + OPERATION_TO_SPAN_NAME[operationName], + operation?.span.id, + ); + return spans.find((candidate) => candidate.output !== undefined) ?? spans[0]; +} + +function spanTreeEvents(events: CapturedLogEvent[]): CapturedLogEvent[] { + const root = findLatestSpan(events, ROOT_NAME); + const items: CapturedLogEvent[] = root ? [root] : []; + + for (const operationName of Object.keys(OPERATION_TO_SPAN_NAME) as Array< + keyof typeof OPERATION_TO_SPAN_NAME + >) { + const operation = findLatestSpan(events, operationName); + const span = findBedrockSpan(events, operationName); + if (operation) { + items.push(operation); + } + if (span) { + items.push(span); + } + } + + return items; +} + +export function defineBedrockRuntimeInstrumentationAssertions(options: { + name: string; + runScenario: RunBedrockRuntimeScenario; + snapshotName: string; + testFileUrl: string; + timeoutMs: number; +}): void { + const spanSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-tree.json`, + ); + const testConfig = { + timeout: options.timeoutMs, + }; + + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + }); + }, options.timeoutMs); + + test("captures the scenario root span", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + expect(root).toBeDefined(); + expect(root?.row.metadata).toMatchObject({ + scenario: SCENARIO_NAME, + }); + }); + + test("captures Bedrock Runtime command spans", testConfig, () => { + for (const operationName of Object.keys(OPERATION_TO_SPAN_NAME) as Array< + keyof typeof OPERATION_TO_SPAN_NAME + >) { + const span = findBedrockSpan(events, operationName); + expect(span, operationName).toBeDefined(); + expect(span?.row.metadata).toMatchObject({ + model: MODEL, + provider: "aws-bedrock", + }); + expect(span?.output).toBeDefined(); + } + }); + + test("captures token metrics for Converse calls", testConfig, () => { + const converseSpan = findBedrockSpan( + events, + "bedrock-converse-operation", + ); + const streamSpan = findBedrockSpan( + events, + "bedrock-converse-stream-operation", + ); + + expect(converseSpan?.metrics).toMatchObject({ + completion_tokens: expect.any(Number), + prompt_tokens: expect.any(Number), + tokens: expect.any(Number), + }); + expect(streamSpan?.metrics).toMatchObject({ + completion_tokens: expect.any(Number), + prompt_tokens: expect.any(Number), + time_to_first_token: expect.any(Number), + tokens: expect.any(Number), + }); + }); + + test("matches span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(spanTreeEvents(events), spanSnapshotPath); + }); + }); +} diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/cassette-filter.mjs b/e2e/scenarios/bedrock-runtime-instrumentation/cassette-filter.mjs new file mode 100644 index 000000000..ab93026de --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/cassette-filter.mjs @@ -0,0 +1,42 @@ +// @ts-check +/** @type {import("@braintrust/seinfeld").FilterSpec} */ +export const filter = [ + "default", + { + ignoreHeaders: [ + "amz-sdk-invocation-id", + "amz-sdk-request", + "x-amz-date", + "x-amz-security-token", + "x-amzn-bedrock-trace", + ], + }, +]; + +/** @type {import("@braintrust/seinfeld").RedactionSpec} */ +export const redact = [ + "paranoid", + { + redactResponse(response) { + return { + ...response, + headers: redactAwsResponseHeaders(response.headers), + }; + }, + }, +]; + +function redactAwsResponseHeaders(headers) { + if (!headers || typeof headers !== "object") { + return headers; + } + + return Object.fromEntries( + Object.entries(headers).map(([key, value]) => [ + key, + key.toLowerCase() === "date" || key.toLowerCase() === "x-amzn-requestid" + ? "[REDACTED]" + : value, + ]), + ); +} diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/constants.mjs b/e2e/scenarios/bedrock-runtime-instrumentation/constants.mjs new file mode 100644 index 000000000..1090b591e --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/constants.mjs @@ -0,0 +1,6 @@ +export const MODEL = + process.env.BRAINTRUST_BEDROCK_CONVERSE_MODEL ?? "us.amazon.nova-lite-v1:0"; +export const REGION = + process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? "us-east-1"; +export const ROOT_NAME = "bedrock-runtime-instrumentation-root"; +export const SCENARIO_NAME = "bedrock-runtime-instrumentation"; diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/package.json b/e2e/scenarios/bedrock-runtime-instrumentation/package.json new file mode 100644 index 000000000..7a931e7c2 --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/package.json @@ -0,0 +1,15 @@ +{ + "name": "@braintrust/e2e-bedrock-runtime-instrumentation", + "private": true, + "braintrustScenario": { + "canary": { + "dependencies": { + "@aws-sdk/client-bedrock-runtime": "latest" + } + } + }, + "dependencies": { + "@aws-sdk/client-bedrock-runtime": "3.1048.0", + "@smithy/node-http-handler": "4.8.1" + } +} diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/pnpm-lock.yaml b/e2e/scenarios/bedrock-runtime-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..bc32ae0b1 --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/pnpm-lock.yaml @@ -0,0 +1,486 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@aws-sdk/client-bedrock-runtime': + specifier: 3.1048.0 + version: 3.1048.0 + '@smithy/node-http-handler': + specifier: 4.8.1 + version: 4.8.1 + +packages: + + '@aws-crypto/crc32@5.2.0': + resolution: {integrity: sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/sha256-browser@5.2.0': + resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} + + '@aws-crypto/sha256-js@5.2.0': + resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/supports-web-crypto@5.2.0': + resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==} + + '@aws-crypto/util@5.2.0': + resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==} + + '@aws-sdk/client-bedrock-runtime@3.1048.0': + resolution: {integrity: sha512-u+NT61JZEkRFtpL0CAw1N1dwxnaLgwVXQl/zjJxTGgLyS/jTIdg2SdoEoCTHxgDyCnqa1HEi9QOoE9/pYRNpOQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.974.22': + resolution: {integrity: sha512-YofH63shc6YRdXjz80BJkpJW+Bkn0Cuu2dn4Rv7s9G2Idt58tgtzQEWxrR2xVljlVfIBeUjPuULnSVYLke3sUQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.48': + resolution: {integrity: sha512-h6FEC95fbexUd6zxm4PdgS82bTcI2PRtUb2ZwMipb/Xr8bPwtf0G8rBo2jp7NA24Mbx2JA8/WingiYpA9RCCyw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.50': + resolution: {integrity: sha512-lJO3OLpjvz5m/RSBQmsG/CEUGsvCy5ruxKwPQaOCqxqCMuyYT2BZwQUTDZVVwqQ9LrZKuK24JSa6r31hL/tvkg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.972.55': + resolution: {integrity: sha512-TBoF4buBGYhXjdZAryayY2TrkQj2B2KfE/msG4V53XCt+w0EhEwM2JRjx8p2grJ2C6gtH5++SAwEvGMRdi0yyw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.54': + resolution: {integrity: sha512-hBWI3wZTdTGiuMfmPts6AWbAjFfRniOQnqx68tc2cQvRKWawFbN9wkLOVPWM1FAOyowZU73mC6Fi+rHSHNyLFw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.57': + resolution: {integrity: sha512-u6dClpzNdWf1HGWz4wwhdXi1wiOofCLniM9S4BQQGlLAN9TW7VB+ld5V533GdKrYMaFeBGFqKnj0JCYvynLqwQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.48': + resolution: {integrity: sha512-w6VZwojPt12WnEkAUy6Nu4K6sWCbBmR7QX390b0nE6vRvkXbrYr9Lq9VySGkfjiMjpUA87op+J4EgvRmtWIDoQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.972.54': + resolution: {integrity: sha512-23uZpIpF2SIFDCa1fcWa202tK4gGeyvX6GIIAjiB8WBsvsVRBMnJ/7dCxHzxf7eZT7GToJg837LDIBnZsl/VUg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.54': + resolution: {integrity: sha512-0Iv5QttS6wcATlodYKgvQj6B9Db51rx7NU9fqu0PoLeS4BIgdYMc/QK4smwLwpm5RFrs02V/eLyEFp3FklvlNQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/eventstream-handler-node@3.972.22': + resolution: {integrity: sha512-tqPJv0dz4+O0hWGm1a6YekcMZyPhDFs/zH73Von7icaVT5n0Jqvm86typ3jRrG+qoUdPhALOnboRLTmnWQTlYQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-eventstream@3.972.18': + resolution: {integrity: sha512-OHpk8YoZi3yexPq8aFt1vN1IxA2zLKvsIR5GpWYylX/ve6kQmY7wxHNSFy/D3t2apMZ16rs76Co4dJWcDyIk3A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-websocket@3.972.30': + resolution: {integrity: sha512-kH6N4f/Fzi9r/dYap8EQ+Zk4NOz8pl4AtWKhzAoG2C1/4YkIHok9APp/e+75woreWQq264n+LkrJsJVZ0Q+M1Q==} + engines: {node: '>= 14.0.0'} + + '@aws-sdk/nested-clients@3.997.22': + resolution: {integrity: sha512-4IwtcYSxEIVw5hcp8ogq0CMbFNZFw7jJUetpfFUhFFeqsa1K8j2Ihg2hnxLyOp3stMZnXda6VzOmPi1AFZQXcg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.35': + resolution: {integrity: sha512-6L/VWs+Wch2stHemCGTmUNqKLMzURxQDK5boNG3Jn3kAOp71meDUuS5sbObpEvFxHDq0uWeSLFDNSYsjNt+Dlg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1048.0': + resolution: {integrity: sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1071.0': + resolution: {integrity: sha512-4LDW2Qob6LoLFuqYSYZq2AyTE9koSE9+i+n5UZcm10GpmQOK0zRD9L4uYlzItiTKksIWgC/qMFChAi3RvKYtMg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.973.13': + resolution: {integrity: sha512-pEHZqRkAlHfnfAU9tK+WpKv/gBNjGJrHMgA3A0iYRGyswBS2t0pfez+lWlwktb3Bqa0ovh7w/QJTFwp3fDxLNg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/util-locate-window@3.965.8': + resolution: {integrity: sha512-uUbMs1cBZPafD0ohUj6EwNf0fPZ534NvBxHox4hjX+0Rxq5paSYUem7+hi833pYrzrcnBATKIYpR02MDXT5M9g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.30': + resolution: {integrity: sha512-StElZPEoBquWwNqw1AcfpzEyZqJvFxouG+mpDNYlcH6ZOrqd2CuIryv+8LV8gNHZUOyKyJF3Dq9vxaXEmDR9TQ==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.2.4': + resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==} + engines: {node: '>=18.0.0'} + + '@nodable/entities@2.2.0': + resolution: {integrity: sha512-9uGyhaQavEUMC8AIddIjau4NsnsXhou+j5sBAGojCM1oxmQpVKTWR/9JxABD6UAv12vpIms55fPZKFQEhG6uBg==} + + '@smithy/core@3.25.1': + resolution: {integrity: sha512-zpDbpXBCBsxfLtG2GEUyfgvHvSFrw5CwDZSNzL0v52gx/c3oPlPbm+7W7num8xs6vyiUBn+bvYPHcQDOXZynCQ==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.4.1': + resolution: {integrity: sha512-TSAF5NHgxEsllbErYWbK8aLnl5L601NGc5VYJlSPsKnf3YlkhdoBN+geGcaU00oiw2OK3QO5LA3QNXiiWhCidQ==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.5.1': + resolution: {integrity: sha512-96JrD1q71anokymx9Iblb+zKmNQYNstlV/25A9ZYIJ2A0rp1r7/GZAIm0bDWSmVvz3DpNOCZuabzsiL+w0UHhw==} + engines: {node: '>=18.0.0'} + + '@smithy/is-array-buffer@2.2.0': + resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==} + engines: {node: '>=14.0.0'} + + '@smithy/node-http-handler@4.8.1': + resolution: {integrity: sha512-emtXvoky671puri18ETf64AFIQUGIEA093F2drXpBgB0OGnBLjcwNR3CA2mYu62IAqNsS56xa5lnTxAgPq7cjw==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.5.1': + resolution: {integrity: sha512-X9rVls3En0z3NtrmguTmpRM0/NqtWUxBjal6fcAkwtsub+gOdLZ6kD+V7xhUgFMGdG14bHbZ7M5QjaRI1+DatQ==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.15.0': + resolution: {integrity: sha512-Z5TAOxygoFvybJV3igo5SloFflSokHx2hu1eFA+DxDTcn+FtKxUSui+rbTRG1pAafMA888Z3MVvCWUuvCrTXjg==} + engines: {node: '>=18.0.0'} + + '@smithy/util-buffer-from@2.2.0': + resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==} + engines: {node: '>=14.0.0'} + + '@smithy/util-utf8@2.3.0': + resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==} + engines: {node: '>=14.0.0'} + + anynum@1.0.1: + resolution: {integrity: sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==} + + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + + fast-xml-builder@1.2.0: + resolution: {integrity: sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==} + + fast-xml-parser@5.7.3: + resolution: {integrity: sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==} + hasBin: true + + path-expression-matcher@1.5.0: + resolution: {integrity: sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==} + engines: {node: '>=14.0.0'} + + strnum@2.4.1: + resolution: {integrity: sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg==} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + xml-naming@0.1.0: + resolution: {integrity: sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==} + engines: {node: '>=16.0.0'} + +snapshots: + + '@aws-crypto/crc32@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.13 + tslib: 2.8.1 + + '@aws-crypto/sha256-browser@5.2.0': + dependencies: + '@aws-crypto/sha256-js': 5.2.0 + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.13 + '@aws-sdk/util-locate-window': 3.965.8 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-crypto/sha256-js@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.13 + tslib: 2.8.1 + + '@aws-crypto/supports-web-crypto@5.2.0': + dependencies: + tslib: 2.8.1 + + '@aws-crypto/util@5.2.0': + dependencies: + '@aws-sdk/types': 3.973.13 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-sdk/client-bedrock-runtime@3.1048.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.22 + '@aws-sdk/credential-provider-node': 3.972.57 + '@aws-sdk/eventstream-handler-node': 3.972.22 + '@aws-sdk/middleware-eventstream': 3.972.18 + '@aws-sdk/middleware-websocket': 3.972.30 + '@aws-sdk/token-providers': 3.1048.0 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/fetch-http-handler': 5.5.1 + '@smithy/node-http-handler': 4.8.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/core@3.974.22': + dependencies: + '@aws-sdk/types': 3.973.13 + '@aws-sdk/xml-builder': 3.972.30 + '@aws/lambda-invoke-store': 0.2.4 + '@smithy/core': 3.25.1 + '@smithy/signature-v4': 5.5.1 + '@smithy/types': 4.15.0 + bowser: 2.14.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.48': + dependencies: + '@aws-sdk/core': 3.974.22 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.50': + dependencies: + '@aws-sdk/core': 3.974.22 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/fetch-http-handler': 5.5.1 + '@smithy/node-http-handler': 4.8.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.972.55': + dependencies: + '@aws-sdk/core': 3.974.22 + '@aws-sdk/credential-provider-env': 3.972.48 + '@aws-sdk/credential-provider-http': 3.972.50 + '@aws-sdk/credential-provider-login': 3.972.54 + '@aws-sdk/credential-provider-process': 3.972.48 + '@aws-sdk/credential-provider-sso': 3.972.54 + '@aws-sdk/credential-provider-web-identity': 3.972.54 + '@aws-sdk/nested-clients': 3.997.22 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/credential-provider-imds': 4.4.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.54': + dependencies: + '@aws-sdk/core': 3.974.22 + '@aws-sdk/nested-clients': 3.997.22 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.57': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.48 + '@aws-sdk/credential-provider-http': 3.972.50 + '@aws-sdk/credential-provider-ini': 3.972.55 + '@aws-sdk/credential-provider-process': 3.972.48 + '@aws-sdk/credential-provider-sso': 3.972.54 + '@aws-sdk/credential-provider-web-identity': 3.972.54 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/credential-provider-imds': 4.4.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.48': + dependencies: + '@aws-sdk/core': 3.974.22 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.972.54': + dependencies: + '@aws-sdk/core': 3.974.22 + '@aws-sdk/nested-clients': 3.997.22 + '@aws-sdk/token-providers': 3.1071.0 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.54': + dependencies: + '@aws-sdk/core': 3.974.22 + '@aws-sdk/nested-clients': 3.997.22 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/eventstream-handler-node@3.972.22': + dependencies: + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-eventstream@3.972.18': + dependencies: + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-websocket@3.972.30': + dependencies: + '@aws-sdk/core': 3.974.22 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/fetch-http-handler': 5.5.1 + '@smithy/signature-v4': 5.5.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.22': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.22 + '@aws-sdk/signature-v4-multi-region': 3.996.35 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/fetch-http-handler': 5.5.1 + '@smithy/node-http-handler': 4.8.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.35': + dependencies: + '@aws-sdk/types': 3.973.13 + '@smithy/signature-v4': 5.5.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1048.0': + dependencies: + '@aws-sdk/core': 3.974.22 + '@aws-sdk/nested-clients': 3.997.22 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1071.0': + dependencies: + '@aws-sdk/core': 3.974.22 + '@aws-sdk/nested-clients': 3.997.22 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/types@3.973.13': + dependencies: + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@aws-sdk/util-locate-window@3.965.8': + dependencies: + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.30': + dependencies: + '@smithy/types': 4.15.0 + fast-xml-parser: 5.7.3 + tslib: 2.8.1 + + '@aws/lambda-invoke-store@0.2.4': {} + + '@nodable/entities@2.2.0': {} + + '@smithy/core@3.25.1': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.4.1': + dependencies: + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.5.1': + dependencies: + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@smithy/is-array-buffer@2.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/node-http-handler@4.8.1': + dependencies: + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@smithy/signature-v4@5.5.1': + dependencies: + '@smithy/core': 3.25.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + + '@smithy/types@4.15.0': + dependencies: + tslib: 2.8.1 + + '@smithy/util-buffer-from@2.2.0': + dependencies: + '@smithy/is-array-buffer': 2.2.0 + tslib: 2.8.1 + + '@smithy/util-utf8@2.3.0': + dependencies: + '@smithy/util-buffer-from': 2.2.0 + tslib: 2.8.1 + + anynum@1.0.1: {} + + bowser@2.14.1: {} + + fast-xml-builder@1.2.0: + dependencies: + path-expression-matcher: 1.5.0 + xml-naming: 0.1.0 + + fast-xml-parser@5.7.3: + dependencies: + '@nodable/entities': 2.2.0 + fast-xml-builder: 1.2.0 + path-expression-matcher: 1.5.0 + strnum: 2.4.1 + + path-expression-matcher@1.5.0: {} + + strnum@2.4.1: + dependencies: + anynum: 1.0.1 + + tslib@2.8.1: {} + + xml-naming@0.1.0: {} diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/scenario.impl.mjs b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.impl.mjs new file mode 100644 index 000000000..185dacc5d --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.impl.mjs @@ -0,0 +1,152 @@ +import { wrapBedrockRuntime } from "braintrust"; +import { + collectAsync, + runOperation, + runTracedScenario, +} from "../../helpers/provider-runtime.mjs"; +import { MODEL, REGION, ROOT_NAME, SCENARIO_NAME } from "./constants.mjs"; + +export const BEDROCK_RUNTIME_SCENARIO_TIMEOUT_MS = 180_000; + +function novaMessageBody(text) { + return { + schemaVersion: "messages-v1", + messages: [ + { + role: "user", + content: [{ text }], + }, + ], + inferenceConfig: { + maxTokens: 16, + temperature: 0, + topP: 0.9, + }, + }; +} + +function bedrockClientConfig(NodeHttpHandler) { + const endpoint = + process.env.AWS_BEDROCK_RUNTIME_BASE_URL ?? + process.env.BEDROCK_RUNTIME_BASE_URL; + return { + region: REGION, + requestHandler: new NodeHttpHandler({ + requestTimeout: BEDROCK_RUNTIME_SCENARIO_TIMEOUT_MS, + }), + ...(endpoint ? { endpoint } : {}), + }; +} + +function assertBedrockAuthEnv() { + if (!process.env.AWS_BEARER_TOKEN_BEDROCK) { + throw new Error("Expected AWS_BEARER_TOKEN_BEDROCK to be set for e2e"); + } +} + +export async function runBedrockRuntimeInstrumentationScenario(options) { + assertBedrockAuthEnv(); + + const baseClient = new options.BedrockRuntimeClient( + bedrockClientConfig(options.NodeHttpHandler), + ); + const client = options.decorateClient + ? options.decorateClient(baseClient) + : baseClient; + + await runTracedScenario({ + callback: async () => { + await runOperation("bedrock-converse-operation", "converse", async () => { + await client.send( + new options.ConverseCommand({ + inferenceConfig: { + maxTokens: 16, + temperature: 0, + topP: 0.9, + }, + messages: [ + { + role: "user", + content: [{ text: "Reply with exactly OK." }], + }, + ], + modelId: MODEL, + }), + ); + }); + + await runOperation( + "bedrock-converse-stream-operation", + "converse-stream", + async () => { + const response = await client.send( + new options.ConverseStreamCommand({ + inferenceConfig: { + maxTokens: 16, + temperature: 0, + topP: 0.9, + }, + messages: [ + { + role: "user", + content: [{ text: "Reply with exactly STREAM." }], + }, + ], + modelId: MODEL, + }), + ); + await collectAsync(response.stream ?? []); + }, + ); + + await runOperation( + "bedrock-invoke-model-operation", + "invoke-model", + async () => { + await client.send( + new options.InvokeModelCommand({ + accept: "application/json", + body: JSON.stringify(novaMessageBody("Reply with exactly RAW.")), + contentType: "application/json", + modelId: MODEL, + }), + ); + }, + ); + + await runOperation( + "bedrock-invoke-model-stream-operation", + "invoke-model-stream", + async () => { + const response = await client.send( + new options.InvokeModelWithResponseStreamCommand({ + accept: "application/json", + body: JSON.stringify( + novaMessageBody("Reply with exactly RAWSTREAM."), + ), + contentType: "application/json", + modelId: MODEL, + }), + ); + await collectAsync(response.body ?? []); + }, + ); + }, + metadata: { + scenario: SCENARIO_NAME, + }, + projectNameBase: "e2e-bedrock-runtime-instrumentation", + rootName: ROOT_NAME, + }); +} + +export async function runWrappedBedrockRuntimeInstrumentation(options) { + await runBedrockRuntimeInstrumentationScenario({ + decorateClient: wrapBedrockRuntime, + ...options, + }); +} + +export async function runAutoBedrockRuntimeInstrumentation(options) { + await runBedrockRuntimeInstrumentationScenario(options); +} diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/scenario.mjs b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.mjs new file mode 100644 index 000000000..13b7b1f37 --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.mjs @@ -0,0 +1,21 @@ +import { + BedrockRuntimeClient, + ConverseCommand, + ConverseStreamCommand, + InvokeModelCommand, + InvokeModelWithResponseStreamCommand, +} from "@aws-sdk/client-bedrock-runtime"; +import { NodeHttpHandler } from "@smithy/node-http-handler"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoBedrockRuntimeInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => { + await runAutoBedrockRuntimeInstrumentation({ + BedrockRuntimeClient, + ConverseCommand, + ConverseStreamCommand, + InvokeModelCommand, + InvokeModelWithResponseStreamCommand, + NodeHttpHandler, + }); +}); diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/scenario.test.ts b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.test.ts new file mode 100644 index 000000000..1133f4abb --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.test.ts @@ -0,0 +1,56 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineBedrockRuntimeInstrumentationAssertions } from "./assertions"; +import { BEDROCK_RUNTIME_SCENARIO_TIMEOUT_MS } from "./scenario.impl.mjs"; + +const originalScenarioDir = resolveScenarioDir(import.meta.url); +const scenarioDir = await prepareScenarioDir({ + scenarioDir: originalScenarioDir, +}); +const bedrockRuntimeSdkVersion = await readInstalledPackageVersion( + scenarioDir, + "@aws-sdk/client-bedrock-runtime", +); + +describe(`bedrock runtime sdk ${bedrockRuntimeSdkVersion}`, () => { + defineBedrockRuntimeInstrumentationAssertions({ + name: "wrapped instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: "scenario.ts", + runContext: { + variantKey: "bedrock-runtime-v3-1048-wrapped", + originalScenarioDir, + }, + scenarioDir, + timeoutMs: BEDROCK_RUNTIME_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: "bedrock-runtime-v3-1048-wrapped", + testFileUrl: import.meta.url, + timeoutMs: BEDROCK_RUNTIME_SCENARIO_TIMEOUT_MS, + }); + + defineBedrockRuntimeInstrumentationAssertions({ + name: "auto-hook instrumentation", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: "scenario.mjs", + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext: { + variantKey: "bedrock-runtime-v3-1048-auto", + originalScenarioDir, + }, + scenarioDir, + timeoutMs: BEDROCK_RUNTIME_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: "bedrock-runtime-v3-1048-auto", + testFileUrl: import.meta.url, + timeoutMs: BEDROCK_RUNTIME_SCENARIO_TIMEOUT_MS, + }); +}); diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/scenario.ts b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.ts new file mode 100644 index 000000000..f0557f65b --- /dev/null +++ b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.ts @@ -0,0 +1,21 @@ +import { + BedrockRuntimeClient, + ConverseCommand, + ConverseStreamCommand, + InvokeModelCommand, + InvokeModelWithResponseStreamCommand, +} from "@aws-sdk/client-bedrock-runtime"; +import { NodeHttpHandler } from "@smithy/node-http-handler"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runWrappedBedrockRuntimeInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => { + await runWrappedBedrockRuntimeInstrumentation({ + BedrockRuntimeClient, + ConverseCommand, + ConverseStreamCommand, + InvokeModelCommand, + InvokeModelWithResponseStreamCommand, + NodeHttpHandler, + }); +}); diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.json index b08553edc..4e4b2a50c 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.txt index 3540d5d48..ab82af0eb 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.json index b08553edc..4e4b2a50c 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.txt index 3540d5d48..ab82af0eb 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/js/src/auto-instrumentations/configs/all.ts b/js/src/auto-instrumentations/configs/all.ts index ad179de34..b92d67b5b 100644 --- a/js/src/auto-instrumentations/configs/all.ts +++ b/js/src/auto-instrumentations/configs/all.ts @@ -6,6 +6,7 @@ import { } from "../../instrumentation/config"; import { aiSDKConfigs } from "./ai-sdk"; import { anthropicConfigs } from "./anthropic"; +import { bedrockRuntimeConfigs } from "./bedrock-runtime"; import { claudeAgentSDKConfigs } from "./claude-agent-sdk"; import { cohereConfigs } from "./cohere"; import { cursorSDKConfigs } from "./cursor-sdk"; @@ -38,6 +39,10 @@ const defaultInstrumentationConfigGroups: readonly InstrumentationConfigGroup[] configs: openAICodexConfigs, }, { integrations: ["anthropic"], configs: anthropicConfigs }, + { + integrations: ["bedrock", "awsBedrock", "awsBedrockRuntime"], + configs: bedrockRuntimeConfigs, + }, { integrations: ["aisdk", "vercel"], configs: aiSDKConfigs, diff --git a/js/src/auto-instrumentations/configs/bedrock-runtime.test.ts b/js/src/auto-instrumentations/configs/bedrock-runtime.test.ts new file mode 100644 index 000000000..bd36d7361 --- /dev/null +++ b/js/src/auto-instrumentations/configs/bedrock-runtime.test.ts @@ -0,0 +1,88 @@ +import { create, type ModuleType } from "@apm-js-collab/code-transformer"; +import { describe, expect, it } from "vitest"; +import { bedrockRuntimeConfigs } from "./bedrock-runtime"; + +const CJS_CLIENT_SOURCE = ` +class Client { + send(command, optionsOrCb, cb) { + if (typeof optionsOrCb === "function" || typeof cb === "function") { + return undefined; + } + return Promise.resolve(command); + } +} +module.exports = { Client }; +`; + +const ESM_CLIENT_SOURCE = ` +export class Client { + send(command, optionsOrCb, cb) { + if (typeof optionsOrCb === "function" || typeof cb === "function") { + return undefined; + } + return Promise.resolve(command); + } +} +`; + +describe("bedrockRuntimeConfigs", () => { + it("matches current and legacy Smithy Client.send entry files", () => { + const matcher = create(bedrockRuntimeConfigs); + + try { + for (const entry of [ + { + channelName: "orchestrion:@smithy/core:client.send", + moduleName: "@smithy/core", + moduleType: "cjs" as ModuleType, + path: "dist-cjs/submodules/client/index.js", + source: CJS_CLIENT_SOURCE, + version: "3.25.1", + }, + { + channelName: "orchestrion:@smithy/core:client.send", + moduleName: "@smithy/core", + moduleType: "esm" as ModuleType, + path: "dist-es/submodules/client/smithy-client/client.js", + source: ESM_CLIENT_SOURCE, + version: "3.25.1", + }, + { + channelName: "orchestrion:@smithy/smithy-client:client.send", + moduleName: "@smithy/smithy-client", + moduleType: "cjs" as ModuleType, + path: "dist-cjs/index.js", + source: CJS_CLIENT_SOURCE, + version: "4.8.0", + }, + { + channelName: "orchestrion:@smithy/smithy-client:client.send", + moduleName: "@smithy/smithy-client", + moduleType: "esm" as ModuleType, + path: "dist-es/client.js", + source: ESM_CLIENT_SOURCE, + version: "4.8.0", + }, + ]) { + const transformer = matcher.getTransformer( + entry.moduleName, + entry.version, + entry.path, + ); + + try { + expect(transformer).toBeDefined(); + const transformed = transformer!.transform( + entry.source, + entry.moduleType, + ); + expect(transformed.code).toContain(entry.channelName); + } finally { + transformer?.free(); + } + } + } finally { + matcher.free(); + } + }); +}); diff --git a/js/src/auto-instrumentations/configs/bedrock-runtime.ts b/js/src/auto-instrumentations/configs/bedrock-runtime.ts new file mode 100644 index 000000000..6100e91eb --- /dev/null +++ b/js/src/auto-instrumentations/configs/bedrock-runtime.ts @@ -0,0 +1,60 @@ +import type { InstrumentationConfig } from "@apm-js-collab/code-transformer"; +import { + smithyClientChannels, + smithyCoreChannels, +} from "../../instrumentation/plugins/bedrock-runtime-channels"; + +export const bedrockRuntimeConfigs: InstrumentationConfig[] = [ + { + channelName: smithyCoreChannels.clientSend.channelName, + module: { + name: "@smithy/core", + versionRange: ">=3.0.0 <4.0.0", + filePath: "dist-cjs/submodules/client/index.js", + }, + functionQuery: { + className: "Client", + methodName: "send", + kind: "Async", + }, + }, + { + channelName: smithyCoreChannels.clientSend.channelName, + module: { + name: "@smithy/core", + versionRange: ">=3.0.0 <4.0.0", + filePath: "dist-es/submodules/client/smithy-client/client.js", + }, + functionQuery: { + className: "Client", + methodName: "send", + kind: "Async", + }, + }, + { + channelName: smithyClientChannels.clientSend.channelName, + module: { + name: "@smithy/smithy-client", + versionRange: ">=4.0.0 <5.0.0", + filePath: "dist-cjs/index.js", + }, + functionQuery: { + className: "Client", + methodName: "send", + kind: "Async", + }, + }, + { + channelName: smithyClientChannels.clientSend.channelName, + module: { + name: "@smithy/smithy-client", + versionRange: ">=4.0.0 <5.0.0", + filePath: "dist-es/client.js", + }, + functionQuery: { + className: "Client", + methodName: "send", + kind: "Async", + }, + }, +]; diff --git a/js/src/auto-instrumentations/index.ts b/js/src/auto-instrumentations/index.ts index e316649c0..017a7064c 100644 --- a/js/src/auto-instrumentations/index.ts +++ b/js/src/auto-instrumentations/index.ts @@ -31,6 +31,7 @@ export { openaiConfigs } from "./configs/openai"; export { openAICodexConfigs } from "./configs/openai-codex"; export { anthropicConfigs } from "./configs/anthropic"; +export { bedrockRuntimeConfigs } from "./configs/bedrock-runtime"; export { aiSDKConfigs } from "./configs/ai-sdk"; export { claudeAgentSDKConfigs } from "./configs/claude-agent-sdk"; export { cursorSDKConfigs } from "./configs/cursor-sdk"; diff --git a/js/src/exports.ts b/js/src/exports.ts index 54cd03e2a..db8a54226 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -194,6 +194,7 @@ export { wrapOpenRouter } from "./wrappers/openrouter"; export { wrapMistral } from "./wrappers/mistral"; export { wrapCohere } from "./wrappers/cohere"; export { wrapGroq } from "./wrappers/groq"; +export { wrapBedrockRuntime } from "./wrappers/bedrock-runtime"; export { wrapCopilotClient } from "./wrappers/github-copilot"; export { wrapVitest } from "./wrappers/vitest"; export { initNodeTestSuite } from "./wrappers/node-test"; diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index 36e7e83f0..e80e15776 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -14,6 +14,7 @@ import { MistralPlugin } from "./plugins/mistral-plugin"; import { GoogleADKPlugin } from "./plugins/google-adk-plugin"; import { CoherePlugin } from "./plugins/cohere-plugin"; import { GroqPlugin } from "./plugins/groq-plugin"; +import { BedrockRuntimePlugin } from "./plugins/bedrock-runtime-plugin"; import { GenkitPlugin } from "./plugins/genkit-plugin"; import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin"; import { FluePlugin } from "./plugins/flue-plugin"; @@ -66,6 +67,7 @@ export class BraintrustPlugin extends BasePlugin { private googleADKPlugin: GoogleADKPlugin | null = null; private coherePlugin: CoherePlugin | null = null; private groqPlugin: GroqPlugin | null = null; + private bedrockRuntimePlugin: BedrockRuntimePlugin | null = null; private genkitPlugin: GenkitPlugin | null = null; private gitHubCopilotPlugin: GitHubCopilotPlugin | null = null; private fluePlugin: FluePlugin | null = null; @@ -164,6 +166,15 @@ export class BraintrustPlugin extends BasePlugin { this.groqPlugin.enable(); } + if ( + integrations.bedrock !== false && + integrations.awsBedrock !== false && + integrations.awsBedrockRuntime !== false + ) { + this.bedrockRuntimePlugin = new BedrockRuntimePlugin(); + this.bedrockRuntimePlugin.enable(); + } + if (integrations.genkit !== false) { this.genkitPlugin = new GenkitPlugin(); this.genkitPlugin.enable(); @@ -272,6 +283,11 @@ export class BraintrustPlugin extends BasePlugin { this.groqPlugin = null; } + if (this.bedrockRuntimePlugin) { + this.bedrockRuntimePlugin.disable(); + this.bedrockRuntimePlugin = null; + } + if (this.genkitPlugin) { this.genkitPlugin.disable(); this.genkitPlugin = null; diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index 299b2b827..976348d1e 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -18,6 +18,9 @@ export interface InstrumentationIntegrationsConfig { mistral?: boolean; cohere?: boolean; groq?: boolean; + bedrock?: boolean; + awsBedrock?: boolean; + awsBedrockRuntime?: boolean; genkit?: boolean; gitHubCopilot?: boolean; openaiCodexSDK?: boolean; @@ -79,6 +82,12 @@ const envIntegrationAliases: Record< cohere: "cohere", groq: "groq", "groq-sdk": "groq", + bedrock: "bedrock", + "aws-bedrock": "awsBedrock", + awsbedrock: "awsBedrock", + "aws-bedrock-runtime": "awsBedrockRuntime", + awsbedrockruntime: "awsBedrockRuntime", + "@aws-sdk/client-bedrock-runtime": "awsBedrockRuntime", genkit: "genkit", "firebase-genkit": "genkit", githubcopilot: "gitHubCopilot", @@ -115,6 +124,9 @@ export function getDefaultInstrumentationIntegrations(): Record< mistral: true, cohere: true, groq: true, + bedrock: true, + awsBedrock: true, + awsBedrockRuntime: true, genkit: true, gitHubCopilot: true, langchain: true, diff --git a/js/src/instrumentation/core/channel-tracing-utils.ts b/js/src/instrumentation/core/channel-tracing-utils.ts index 49cc87561..143e19e45 100644 --- a/js/src/instrumentation/core/channel-tracing-utils.ts +++ b/js/src/instrumentation/core/channel-tracing-utils.ts @@ -1,8 +1,13 @@ import type { ChannelSpanInfo, SpanInfoCarrier, StartEvent } from "./types"; +import { debugLogger } from "../../debug-logger"; import { isObject, mergeDicts } from "../../util"; +type ChannelConfigName = + | string + | ((args: unknown[], event: unknown) => string | undefined); + export type ChannelConfig = { - name: string; + name: ChannelConfigName; shouldTrace?: (args: unknown[], event: unknown) => boolean; type: string; }; @@ -49,7 +54,7 @@ export function buildStartSpanArgs( name: typeof spanInfo?.name === "string" && spanInfo.name ? spanInfo.name - : config.name, + : resolveConfigName(config.name, event), spanAttributes, spanInfoMetadata: isObject(spanInfo?.metadata) ? spanInfo.metadata @@ -57,6 +62,20 @@ export function buildStartSpanArgs( }; } +function resolveConfigName(name: ChannelConfigName, event: StartEvent): string { + if (typeof name === "string") { + return name; + } + + try { + const resolved = name(event.arguments ?? [], event); + return resolved || "unknown"; + } catch (error) { + debugLogger.error("Error resolving instrumentation span name:", error); + return "unknown"; + } +} + export function mergeInputMetadata( metadata: unknown, spanInfoMetadata: Record | undefined, diff --git a/js/src/instrumentation/plugins/bedrock-runtime-channels.ts b/js/src/instrumentation/plugins/bedrock-runtime-channels.ts new file mode 100644 index 000000000..e884b7bd3 --- /dev/null +++ b/js/src/instrumentation/plugins/bedrock-runtime-channels.ts @@ -0,0 +1,34 @@ +import { channel, defineChannels } from "../core/channel-definitions"; +import type { + BedrockRuntimeChannelContext, + BedrockRuntimeCommandLike, + BedrockRuntimeConverseStreamEvent, + BedrockRuntimeResponseStreamEvent, + BedrockRuntimeSendResult, +} from "../../vendor-sdk-types/bedrock-runtime"; + +export type BedrockRuntimeStreamEvent = + | BedrockRuntimeConverseStreamEvent + | BedrockRuntimeResponseStreamEvent; + +const clientSendChannel = channel< + [BedrockRuntimeCommandLike, unknown?], + BedrockRuntimeSendResult, + BedrockRuntimeChannelContext, + BedrockRuntimeStreamEvent +>({ + channelName: "client.send", + kind: "async", +}); + +export const bedrockRuntimeChannels = defineChannels("aws-bedrock-runtime", { + clientSend: clientSendChannel, +}); + +export const smithyCoreChannels = defineChannels("@smithy/core", { + clientSend: clientSendChannel, +}); + +export const smithyClientChannels = defineChannels("@smithy/smithy-client", { + clientSend: clientSendChannel, +}); diff --git a/js/src/instrumentation/plugins/bedrock-runtime-common.ts b/js/src/instrumentation/plugins/bedrock-runtime-common.ts new file mode 100644 index 000000000..c71b56f67 --- /dev/null +++ b/js/src/instrumentation/plugins/bedrock-runtime-common.ts @@ -0,0 +1,81 @@ +import { isObject } from "../../../util/index"; +import type { + BedrockRuntimeCommandLike, + BedrockRuntimeCommandName, +} from "../../vendor-sdk-types/bedrock-runtime"; + +export type BedrockRuntimeOperation = + | "converse" + | "converseStream" + | "invokeModel" + | "invokeModelWithResponseStream"; + +const BEDROCK_RUNTIME_COMMAND_OPERATIONS: Record< + BedrockRuntimeCommandName, + BedrockRuntimeOperation +> = { + ConverseCommand: "converse", + ConverseStreamCommand: "converseStream", + InvokeModelCommand: "invokeModel", + InvokeModelWithResponseStreamCommand: "invokeModelWithResponseStream", +}; + +export function getBedrockRuntimeCommandName( + command: unknown, +): BedrockRuntimeCommandName | undefined { + if (!isObject(command) || !isObject(command.constructor)) { + return undefined; + } + + const input = (command as BedrockRuntimeCommandLike).input; + if (!isObject(input) || typeof input.modelId !== "string") { + return undefined; + } + + const commandName = command.constructor.name; + return isBedrockRuntimeCommandName(commandName) ? commandName : undefined; +} + +export function getBedrockRuntimeOperation( + command: unknown, +): BedrockRuntimeOperation | undefined { + const commandName = getBedrockRuntimeCommandName(command); + return commandName + ? BEDROCK_RUNTIME_COMMAND_OPERATIONS[commandName] + : undefined; +} + +export function getBedrockRuntimeCommandInput( + command: unknown, +): unknown | undefined { + return isObject(command) + ? (command as BedrockRuntimeCommandLike).input + : undefined; +} + +export function buildBedrockRuntimeSpanInfo(command: unknown): { + name: string; + metadata: Record; +} { + const commandName = getBedrockRuntimeCommandName(command); + const operation = getBedrockRuntimeOperation(command); + + return { + name: operation ? `bedrock.${operation}` : "bedrock.client.send", + metadata: { + ...(commandName ? { command: commandName } : {}), + ...(operation ? { operation } : {}), + }, + }; +} + +function isBedrockRuntimeCommandName( + commandName: unknown, +): commandName is BedrockRuntimeCommandName { + return ( + commandName === "ConverseCommand" || + commandName === "ConverseStreamCommand" || + commandName === "InvokeModelCommand" || + commandName === "InvokeModelWithResponseStreamCommand" + ); +} diff --git a/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts b/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts new file mode 100644 index 000000000..3b642163b --- /dev/null +++ b/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts @@ -0,0 +1,209 @@ +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { _exportsForTestingOnly, initLogger } from "../../logger"; +import { configureNode } from "../../node/config"; +import { smithyCoreChannels } from "./bedrock-runtime-channels"; +import { + aggregateBedrockConverseStreamChunks, + parseBedrockRuntimeMetrics, +} from "./bedrock-runtime-plugin"; + +try { + configureNode(); +} catch { + // Best-effort initialization for test environments. +} + +class ConverseCommand { + constructor(public input: Record) {} +} + +describe("BedrockRuntimePlugin", () => { + let backgroundLogger: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger + >; + + beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + }); + + beforeEach(() => { + backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ + projectName: "bedrock-runtime-plugin.test.ts", + projectId: "test-project-id", + }); + }); + + afterEach(() => { + _exportsForTestingOnly.clearTestBackgroundLogger(); + }); + + it("traces promise-style Smithy send events and ignores callback overloads", async () => { + const tracingChannel = smithyCoreChannels.clientSend.tracingChannel(); + + await smithyCoreChannels.clientSend.tracePromise( + async () => ({ + output: { + message: { + role: "assistant", + content: [{ text: "OK" }], + }, + }, + usage: { + inputTokens: 1, + outputTokens: 1, + totalTokens: 2, + }, + }), + { + arguments: [ + new ConverseCommand({ + messages: [{ role: "user", content: [{ text: "OK" }] }], + modelId: "us.amazon.nova-lite-v1:0", + }), + ], + }, + ); + + const callbackEvent: any = { + arguments: [ + new ConverseCommand({ + messages: [{ role: "user", content: [{ text: "ignored" }] }], + modelId: "us.amazon.nova-lite-v1:0", + }), + () => {}, + ], + }; + tracingChannel.start.publish(callbackEvent); + callbackEvent.result = { + output: { + message: { + role: "assistant", + content: [{ text: "ignored" }], + }, + }, + }; + tracingChannel.asyncEnd.publish(callbackEvent); + + const spans = await backgroundLogger.drain(); + expect(spans).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + span_attributes: expect.objectContaining({ + name: "bedrock.converse", + }), + output: { + role: "assistant", + content: [{ text: "OK" }], + }, + }), + ]), + ); + expect(spans).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + output: { + role: "assistant", + content: [{ text: "ignored" }], + }, + }), + ]), + ); + }); +}); + +describe("parseBedrockRuntimeMetrics", () => { + it("maps Bedrock usage and latency metrics to Braintrust metrics", () => { + expect( + parseBedrockRuntimeMetrics( + { + cacheReadInputTokens: 3, + cacheWriteInputTokens: 2, + inputTokens: 10, + outputTokens: 4, + totalTokens: 14, + }, + { + latencyMs: 321, + }, + ), + ).toEqual({ + completion_tokens: 4, + latency_ms: 321, + prompt_cache_creation_tokens: 2, + prompt_cached_tokens: 3, + prompt_tokens: 10, + tokens: 14, + }); + }); + + it("returns an empty object for unknown values", () => { + expect(parseBedrockRuntimeMetrics(undefined, undefined)).toEqual({}); + expect(parseBedrockRuntimeMetrics({}, {})).toEqual({}); + }); +}); + +describe("aggregateBedrockConverseStreamChunks", () => { + it("aggregates text, stop reason, and metrics from Converse stream events", () => { + expect( + aggregateBedrockConverseStreamChunks([ + { + messageStart: { + role: "assistant", + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { + text: "ST", + }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { + text: "REAM", + }, + }, + }, + { + messageStop: { + stopReason: "end_turn", + }, + }, + { + metadata: { + metrics: { + latencyMs: 456, + }, + usage: { + inputTokens: 6, + outputTokens: 1, + totalTokens: 7, + }, + }, + }, + ]), + ).toEqual({ + metadata: { + stopReason: "end_turn", + }, + metrics: { + completion_tokens: 1, + latency_ms: 456, + prompt_tokens: 6, + tokens: 7, + }, + output: { + content: [ + { + text: "STREAM", + }, + ], + role: "assistant", + }, + }); + }); +}); diff --git a/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts b/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts new file mode 100644 index 000000000..175a1da6e --- /dev/null +++ b/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts @@ -0,0 +1,621 @@ +import { BasePlugin } from "../core"; +import { traceStreamingChannel, unsubscribeAll } from "../core/channel-tracing"; +import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { getCurrentUnixTimestamp } from "../../util"; +import type { Span } from "../../logger"; +import type { AnyAsyncChannel } from "../core/channel-definitions"; +import type { + BedrockRuntimeConverseRequest, + BedrockRuntimeConverseResponse, + BedrockRuntimeConverseStreamEvent, + BedrockRuntimeInvokeModelRequest, + BedrockRuntimeResponseStreamEvent, + BedrockRuntimeTokenUsage, +} from "../../vendor-sdk-types/bedrock-runtime"; +import { + bedrockRuntimeChannels, + smithyClientChannels, + smithyCoreChannels, +} from "./bedrock-runtime-channels"; +import { + buildBedrockRuntimeSpanInfo, + getBedrockRuntimeCommandInput, + getBedrockRuntimeCommandName, + getBedrockRuntimeOperation, +} from "./bedrock-runtime-common"; + +export class BedrockRuntimePlugin extends BasePlugin { + protected onEnable(): void { + this.unsubscribers.push( + ...[ + bedrockRuntimeChannels.clientSend, + smithyCoreChannels.clientSend, + smithyClientChannels.clientSend, + ].map((channel) => traceBedrockRuntimeClientSendChannel(channel)), + ); + } + + protected onDisable(): void { + this.unsubscribers = unsubscribeAll(this.unsubscribers); + } +} + +function traceBedrockRuntimeClientSendChannel( + channel: AnyAsyncChannel, +): () => void { + return traceStreamingChannel(channel, { + name: ([command]) => buildBedrockRuntimeSpanInfo(command).name, + shouldTrace: ([command, optionsOrCb, cb]) => + getBedrockRuntimeOperation(command) !== undefined && + typeof optionsOrCb !== "function" && + typeof cb !== "function", + type: SpanTypeAttribute.LLM, + extractInput: ([command]) => extractBedrockRuntimeInput(command), + extractOutput: (result, endEvent) => + extractBedrockRuntimeOutput(endEvent?.arguments?.[0], result), + extractMetadata: (result, endEvent) => + extractBedrockRuntimeResponseMetadata(endEvent?.arguments?.[0], result), + extractMetrics: (result) => extractBedrockRuntimeResponseMetrics(result), + patchResult: ({ endEvent, result, span, startTime }) => + patchBedrockRuntimeStreamingResult({ + command: endEvent.arguments?.[0], + result, + span, + startTime, + }), + }); +} + +function extractBedrockRuntimeInput(command: unknown): { + input: unknown; + metadata: Record; +} { + const operation = getBedrockRuntimeOperation(command); + const commandName = getBedrockRuntimeCommandName(command); + const request = getBedrockRuntimeCommandInput(command); + const metadata = { + provider: "aws-bedrock", + ...(commandName ? { command: commandName } : {}), + ...(operation ? { operation } : {}), + ...extractBedrockRuntimeRequestMetadata(request), + }; + + if (operation === "converse" || operation === "converseStream") { + const converseRequest = isObject(request) + ? (request as BedrockRuntimeConverseRequest) + : undefined; + return { + input: sanitizeBedrockValue({ + messages: converseRequest?.messages, + system: converseRequest?.system, + }), + metadata, + }; + } + + if ( + operation === "invokeModel" || + operation === "invokeModelWithResponseStream" + ) { + const invokeRequest = isObject(request) + ? (request as BedrockRuntimeInvokeModelRequest) + : undefined; + return { + input: + parseJsonBody(invokeRequest?.body) ?? + summarizeBody(invokeRequest?.body), + metadata, + }; + } + + return { + input: sanitizeBedrockValue(request), + metadata, + }; +} + +function extractBedrockRuntimeRequestMetadata( + request: unknown, +): Record { + if (!isObject(request)) { + return {}; + } + + const metadata: Record = {}; + for (const key of [ + "modelId", + "contentType", + "accept", + "trace", + "guardrailIdentifier", + "guardrailVersion", + "performanceConfig", + "performanceConfigLatency", + "serviceTier", + ]) { + const value = request[key]; + if (value !== undefined) { + metadata[key === "modelId" ? "model" : key] = value; + } + } + + if (isObject(request.inferenceConfig)) { + Object.assign(metadata, sanitizeBedrockValue(request.inferenceConfig)); + } + + return metadata; +} + +function extractBedrockRuntimeOutput( + command: unknown, + result: unknown, +): unknown { + const operation = getBedrockRuntimeOperation(command); + if (operation === "converse") { + return sanitizeBedrockValue( + (result as BedrockRuntimeConverseResponse | undefined)?.output?.message, + ); + } + + if (operation === "invokeModel") { + const response = isObject(result) ? result : undefined; + return parseJsonBody(response?.body) ?? summarizeBody(response?.body); + } + + return sanitizeBedrockValue(result); +} + +function extractBedrockRuntimeResponseMetadata( + command: unknown, + result: unknown, +): Record | undefined { + const operation = getBedrockRuntimeOperation(command); + if (!isObject(result)) { + return undefined; + } + + const metadata: Record = {}; + for (const key of [ + "stopReason", + "contentType", + "performanceConfig", + "performanceConfigLatency", + "serviceTier", + ]) { + const value = result[key]; + if (value !== undefined) { + metadata[key] = value; + } + } + + if ( + operation === "converse" && + result.additionalModelResponseFields !== undefined + ) { + metadata.additionalModelResponseFields = sanitizeBedrockValue( + result.additionalModelResponseFields, + ); + } + + return Object.keys(metadata).length > 0 ? metadata : undefined; +} + +function extractBedrockRuntimeResponseMetrics( + result: unknown, +): Record { + if (!isObject(result)) { + return {}; + } + + const parsedBody = parseJsonBody(result.body); + if (isObject(parsedBody)) { + const metadata = isObject(parsedBody.metadata) + ? parsedBody.metadata + : undefined; + const metrics = parseBedrockRuntimeMetrics( + parsedBody.usage ?? metadata?.usage, + parsedBody.metrics ?? metadata?.metrics, + ); + if (Object.keys(metrics).length > 0) { + return metrics; + } + } + + return parseBedrockRuntimeMetrics(result.usage, result.metrics); +} + +export function parseBedrockRuntimeMetrics( + usage: unknown, + responseMetrics?: unknown, +): Record { + const metrics: Record = {}; + const usageRecord = isObject(usage) + ? (usage as BedrockRuntimeTokenUsage) + : {}; + + if (typeof usageRecord.inputTokens === "number") { + metrics.prompt_tokens = usageRecord.inputTokens; + } + if (typeof usageRecord.outputTokens === "number") { + metrics.completion_tokens = usageRecord.outputTokens; + } + if (typeof usageRecord.totalTokens === "number") { + metrics.tokens = usageRecord.totalTokens; + } + if (typeof usageRecord.cacheReadInputTokens === "number") { + metrics.prompt_cached_tokens = usageRecord.cacheReadInputTokens; + } + if (typeof usageRecord.cacheWriteInputTokens === "number") { + metrics.prompt_cache_creation_tokens = usageRecord.cacheWriteInputTokens; + } + + if ( + isObject(responseMetrics) && + typeof responseMetrics.latencyMs === "number" + ) { + metrics.latency_ms = responseMetrics.latencyMs; + } + + return metrics; +} + +function patchBedrockRuntimeStreamingResult(args: { + command: unknown; + result: unknown; + span: Span; + startTime: number; +}): boolean { + const operation = getBedrockRuntimeOperation(args.command); + if (!isObject(args.result)) { + return false; + } + + if (operation === "converseStream" && isAsyncIterable(args.result.stream)) { + patchConverseStream(args.result.stream, args.span, args.startTime); + return true; + } + + if ( + operation === "invokeModelWithResponseStream" && + isAsyncIterable(args.result.body) + ) { + patchInvokeModelResponseStream(args.result.body, args.span, args.startTime); + return true; + } + + return false; +} + +function patchConverseStream( + stream: AsyncIterable, + span: Span, + startTime: number, +): void { + let firstChunkTime: number | undefined; + patchStreamIfNeeded(stream, { + onChunk: (chunk) => { + if (firstChunkTime === undefined && isObject(chunk.contentBlockDelta)) { + firstChunkTime = getCurrentUnixTimestamp(); + } + }, + onComplete: (chunks) => { + const aggregated = aggregateBedrockConverseStreamChunks(chunks); + const metrics = { ...aggregated.metrics }; + if (firstChunkTime !== undefined) { + metrics.time_to_first_token = firstChunkTime - startTime; + } + + span.log({ + output: aggregated.output, + ...(aggregated.metadata ? { metadata: aggregated.metadata } : {}), + metrics, + }); + span.end(); + }, + onError: (error) => { + span.log({ error: error.message }); + span.end(); + }, + }); +} + +function patchInvokeModelResponseStream( + stream: AsyncIterable, + span: Span, + startTime: number, +): void { + let firstChunkTime: number | undefined; + patchStreamIfNeeded(stream, { + onChunk: (chunk) => { + if (firstChunkTime === undefined && isObject(chunk.chunk)) { + firstChunkTime = getCurrentUnixTimestamp(); + } + }, + onComplete: (chunks) => { + const aggregated = aggregateInvokeModelResponseStreamChunks(chunks); + const metrics = { ...aggregated.metrics }; + if (firstChunkTime !== undefined) { + metrics.time_to_first_token = firstChunkTime - startTime; + } + + span.log({ + output: aggregated.output, + ...(aggregated.metadata ? { metadata: aggregated.metadata } : {}), + metrics, + }); + span.end(); + }, + onError: (error) => { + span.log({ error: error.message }); + span.end(); + }, + }); +} + +export function aggregateBedrockConverseStreamChunks( + chunks: BedrockRuntimeConverseStreamEvent[], +): { + output: unknown; + metrics: Record; + metadata?: Record; +} { + let role: string | undefined; + let stopReason: string | undefined; + let usage: unknown; + let responseMetrics: unknown; + const contentByIndex = new Map>(); + const metadata: Record = {}; + + for (const chunk of chunks) { + if (typeof chunk.messageStart?.role === "string") { + role = chunk.messageStart.role; + } + + const startIndex = chunk.contentBlockStart?.contentBlockIndex; + if (typeof startIndex === "number") { + contentByIndex.set(startIndex, { + ...(contentByIndex.get(startIndex) ?? {}), + ...sanitizeRecord(chunk.contentBlockStart?.start), + }); + } + + const deltaIndex = chunk.contentBlockDelta?.contentBlockIndex; + const delta = chunk.contentBlockDelta?.delta; + if (typeof deltaIndex === "number" && isObject(delta)) { + const existing = contentByIndex.get(deltaIndex) ?? {}; + contentByIndex.set(deltaIndex, mergeContentBlockDelta(existing, delta)); + } + + if (typeof chunk.messageStop?.stopReason === "string") { + stopReason = chunk.messageStop.stopReason; + } + if (chunk.messageStop?.additionalModelResponseFields !== undefined) { + metadata.additionalModelResponseFields = sanitizeBedrockValue( + chunk.messageStop.additionalModelResponseFields, + ); + } + if (chunk.metadata?.usage !== undefined) { + usage = chunk.metadata.usage; + } + if (chunk.metadata?.metrics !== undefined) { + responseMetrics = chunk.metadata.metrics; + } + if (chunk.metadata?.performanceConfig !== undefined) { + metadata.performanceConfig = sanitizeBedrockValue( + chunk.metadata.performanceConfig, + ); + } + if (chunk.metadata?.serviceTier !== undefined) { + metadata.serviceTier = chunk.metadata.serviceTier; + } + } + + if (stopReason !== undefined) { + metadata.stopReason = stopReason; + } + + const content = [...contentByIndex.entries()] + .sort(([left], [right]) => left - right) + .map(([, value]) => value) + .filter((value) => Object.keys(value).length > 0); + + return { + output: { + role, + content, + }, + metrics: parseBedrockRuntimeMetrics(usage, responseMetrics), + ...(Object.keys(metadata).length > 0 ? { metadata } : {}), + }; +} + +function aggregateInvokeModelResponseStreamChunks( + chunks: BedrockRuntimeResponseStreamEvent[], +): { + output: unknown; + metrics: Record; + metadata?: Record; +} { + const parsedChunks = chunks + .map((chunk) => parseJsonBody(chunk.chunk?.bytes)) + .filter((chunk) => chunk !== undefined); + const text = parsedChunks.map(extractTextFromJsonLike).join(""); + const lastMetadataChunk = parsedChunks + .slice() + .reverse() + .find((chunk) => isObject(chunk?.metadata)); + const metadata = isObject(lastMetadataChunk?.metadata) + ? sanitizeRecord(lastMetadataChunk.metadata) + : undefined; + + return { + output: + text.length > 0 + ? { text } + : { + chunk_count: chunks.length, + chunks: sanitizeBedrockValue(parsedChunks.slice(0, 20)), + }, + metrics: parseBedrockRuntimeMetrics( + isObject(metadata) ? metadata.usage : undefined, + isObject(metadata) ? metadata.metrics : undefined, + ), + ...(metadata ? { metadata } : {}), + }; +} + +function mergeContentBlockDelta( + existing: Record, + delta: Record, +): Record { + const next = { ...existing }; + if (typeof delta.text === "string") { + next.text = `${typeof next.text === "string" ? next.text : ""}${delta.text}`; + } + if (isObject(delta.reasoningContent)) { + const existingReasoning = isObject(next.reasoningContent) + ? next.reasoningContent + : {}; + next.reasoningContent = { + ...existingReasoning, + ...sanitizeRecord(delta.reasoningContent), + ...(typeof delta.reasoningContent.text === "string" + ? { + text: `${typeof existingReasoning.text === "string" ? existingReasoning.text : ""}${delta.reasoningContent.text}`, + } + : {}), + }; + } + if (isObject(delta.toolUse)) { + const existingToolUse = isObject(next.toolUse) ? next.toolUse : {}; + next.toolUse = { + ...existingToolUse, + ...sanitizeRecord(delta.toolUse), + ...(typeof delta.toolUse.input === "string" + ? { + input: `${typeof existingToolUse.input === "string" ? existingToolUse.input : ""}${delta.toolUse.input}`, + } + : {}), + }; + } + + for (const [key, value] of Object.entries(delta)) { + if (key !== "text" && key !== "reasoningContent" && key !== "toolUse") { + next[key] = sanitizeBedrockValue(value); + } + } + + return next; +} + +function parseJsonBody(body: unknown): unknown | undefined { + const text = decodeBodyToString(body); + if (text === undefined || text.length === 0) { + return undefined; + } + + try { + return sanitizeBedrockValue(JSON.parse(text)); + } catch { + return undefined; + } +} + +function summarizeBody(body: unknown): unknown { + if (body === undefined || body === null) { + return undefined; + } + + const text = decodeBodyToString(body); + if (text !== undefined) { + return text.length > 2_000 ? `${text.slice(0, 2_000)}...` : text; + } + + return sanitizeBedrockValue(body); +} + +function decodeBodyToString(body: unknown): string | undefined { + if (typeof body === "string") { + return body; + } + if (body instanceof Uint8Array) { + return new TextDecoder().decode(body); + } + if (body instanceof ArrayBuffer) { + return new TextDecoder().decode(new Uint8Array(body)); + } + return undefined; +} + +function sanitizeRecord(value: unknown): Record { + return isObject(value) + ? (sanitizeBedrockValue(value) as Record) + : {}; +} + +function sanitizeBedrockValue(value: unknown, depth = 0): unknown { + if (value === undefined || value === null) { + return value; + } + if (typeof value !== "object") { + return value; + } + if (value instanceof Uint8Array) { + return { byte_length: value.byteLength }; + } + if (value instanceof ArrayBuffer) { + return { byte_length: value.byteLength }; + } + if (Array.isArray(value)) { + return depth > 20 + ? "[MaxDepth]" + : value.map((item) => sanitizeBedrockValue(item, depth + 1)); + } + if (!isObject(value)) { + return String(value); + } + if (depth > 20) { + return "[MaxDepth]"; + } + + const output: Record = {}; + for (const [key, nested] of Object.entries(value)) { + output[key] = sanitizeBedrockValue(nested, depth + 1); + } + return output; +} + +function extractTextFromJsonLike(value: unknown): string { + if (typeof value === "string") { + return value; + } + if (Array.isArray(value)) { + return value.map(extractTextFromJsonLike).join(""); + } + if (!isObject(value)) { + return ""; + } + + const contentBlockDelta = value.contentBlockDelta; + if (isObject(contentBlockDelta)) { + return extractTextFromJsonLike(contentBlockDelta.delta); + } + if (typeof value.text === "string") { + return value.text; + } + if (typeof value.completion === "string") { + return value.completion; + } + if (typeof value.generation === "string") { + return value.generation; + } + if (Array.isArray(value.content)) { + return value.content.map(extractTextFromJsonLike).join(""); + } + if (Array.isArray(value.output)) { + return value.output.map(extractTextFromJsonLike).join(""); + } + + return ""; +} diff --git a/js/src/vendor-sdk-types/bedrock-runtime.ts b/js/src/vendor-sdk-types/bedrock-runtime.ts new file mode 100644 index 000000000..ab6099cf1 --- /dev/null +++ b/js/src/vendor-sdk-types/bedrock-runtime.ts @@ -0,0 +1,172 @@ +import type { ChannelSpanInfo } from "../instrumentation/core/types"; + +export type BedrockRuntimeCommandName = + | "ConverseCommand" + | "ConverseStreamCommand" + | "InvokeModelCommand" + | "InvokeModelWithResponseStreamCommand"; + +export interface BedrockRuntimeCommandLike { + input?: unknown; + constructor?: { + name?: string; + }; + [key: string]: unknown; +} + +export interface BedrockRuntimeClient { + send: ( + command: BedrockRuntimeCommandLike, + optionsOrCb?: unknown, + cb?: unknown, + ) => Promise | unknown; + [key: string]: unknown; +} + +export interface BedrockRuntimeChannelContext { + span_info?: ChannelSpanInfo; +} + +export interface BedrockRuntimeContentBlock { + text?: string; + image?: unknown; + document?: unknown; + toolUse?: unknown; + toolResult?: unknown; + reasoningContent?: unknown; + [key: string]: unknown; +} + +export interface BedrockRuntimeMessage { + role?: string; + content?: BedrockRuntimeContentBlock[]; + [key: string]: unknown; +} + +export interface BedrockRuntimeConverseRequest { + modelId?: string; + messages?: BedrockRuntimeMessage[]; + system?: BedrockRuntimeContentBlock[]; + inferenceConfig?: Record; + toolConfig?: unknown; + guardrailConfig?: unknown; + additionalModelRequestFields?: unknown; + additionalModelResponseFieldPaths?: unknown; + requestMetadata?: unknown; + performanceConfig?: unknown; + serviceTier?: string; + outputConfig?: unknown; + [key: string]: unknown; +} + +export interface BedrockRuntimeTokenUsage { + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + cacheReadInputTokens?: number; + cacheWriteInputTokens?: number; + [key: string]: unknown; +} + +export interface BedrockRuntimeConverseMetrics { + latencyMs?: number; + [key: string]: unknown; +} + +export interface BedrockRuntimeConverseResponse { + output?: { + message?: BedrockRuntimeMessage; + [key: string]: unknown; + }; + stopReason?: string; + usage?: BedrockRuntimeTokenUsage; + metrics?: BedrockRuntimeConverseMetrics; + additionalModelResponseFields?: unknown; + performanceConfig?: unknown; + serviceTier?: string; + [key: string]: unknown; +} + +export interface BedrockRuntimeConverseStreamResponse { + stream?: AsyncIterable; + [key: string]: unknown; +} + +export interface BedrockRuntimeConverseStreamEvent { + messageStart?: { + role?: string; + [key: string]: unknown; + }; + contentBlockStart?: { + contentBlockIndex?: number; + start?: BedrockRuntimeContentBlock; + [key: string]: unknown; + }; + contentBlockDelta?: { + contentBlockIndex?: number; + delta?: BedrockRuntimeContentBlock; + [key: string]: unknown; + }; + contentBlockStop?: { + contentBlockIndex?: number; + [key: string]: unknown; + }; + messageStop?: { + stopReason?: string; + additionalModelResponseFields?: unknown; + [key: string]: unknown; + }; + metadata?: { + usage?: BedrockRuntimeTokenUsage; + metrics?: BedrockRuntimeConverseMetrics; + trace?: unknown; + performanceConfig?: unknown; + serviceTier?: string; + [key: string]: unknown; + }; + [key: string]: unknown; +} + +export interface BedrockRuntimeInvokeModelRequest { + modelId?: string; + body?: unknown; + contentType?: string; + accept?: string; + trace?: string; + guardrailIdentifier?: string; + guardrailVersion?: string; + performanceConfigLatency?: string; + serviceTier?: string; + [key: string]: unknown; +} + +export interface BedrockRuntimeInvokeModelResponse { + body?: unknown; + contentType?: string; + performanceConfigLatency?: string; + serviceTier?: string; + [key: string]: unknown; +} + +export interface BedrockRuntimeInvokeModelWithResponseStreamResponse { + body?: AsyncIterable; + contentType?: string; + performanceConfigLatency?: string; + serviceTier?: string; + [key: string]: unknown; +} + +export interface BedrockRuntimeResponseStreamEvent { + chunk?: { + bytes?: unknown; + [key: string]: unknown; + }; + [key: string]: unknown; +} + +export type BedrockRuntimeSendResult = + | BedrockRuntimeConverseResponse + | BedrockRuntimeConverseStreamResponse + | BedrockRuntimeInvokeModelResponse + | BedrockRuntimeInvokeModelWithResponseStreamResponse + | unknown; diff --git a/js/src/wrappers/bedrock-runtime.test.ts b/js/src/wrappers/bedrock-runtime.test.ts new file mode 100644 index 000000000..996a1498b --- /dev/null +++ b/js/src/wrappers/bedrock-runtime.test.ts @@ -0,0 +1,240 @@ +import { + afterEach, + beforeAll, + beforeEach, + describe, + expect, + test, + vi, +} from "vitest"; +import { configureNode } from "../node/config"; +import { _exportsForTestingOnly, initLogger } from "../logger"; +import { wrapBedrockRuntime } from "./bedrock-runtime"; + +try { + configureNode(); +} catch { + // Best-effort initialization for test environments. +} + +class ConverseCommand { + constructor(public input: Record) {} +} + +class ConverseStreamCommand { + constructor(public input: Record) {} +} + +class ListFoundationModelsCommand { + constructor(public input: Record) {} +} + +describe("bedrock runtime wrapper", () => { + let backgroundLogger: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger + >; + + beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + }); + + beforeEach(() => { + backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ + projectId: "test-project-id", + projectName: "bedrock-runtime.test.ts", + }); + }); + + afterEach(() => { + _exportsForTestingOnly.clearTestBackgroundLogger(); + vi.restoreAllMocks(); + }); + + test("returns original object for unsupported clients", () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const invalid = { foo: "bar" }; + + expect(wrapBedrockRuntime(invalid)).toBe(invalid); + expect(warnSpy).toHaveBeenCalledWith( + "Unsupported Bedrock Runtime library. Not wrapping.", + ); + }); + + test("wraps supported send operations and preserves callback calls", async () => { + async function* stream() { + yield { + messageStart: { + role: "assistant", + }, + }; + yield { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { + text: "STREAM", + }, + }, + }; + yield { + messageStop: { + stopReason: "end_turn", + }, + }; + yield { + metadata: { + metrics: { + latencyMs: 42, + }, + usage: { + inputTokens: 4, + outputTokens: 1, + totalTokens: 5, + }, + }, + }; + } + + const send = vi.fn( + ( + command: + | ConverseCommand + | ConverseStreamCommand + | ListFoundationModelsCommand, + optionsOrCb?: unknown, + ) => { + if (typeof optionsOrCb === "function") { + optionsOrCb(null, { callback: true }); + return undefined; + } + + if (command instanceof ConverseStreamCommand) { + return Promise.resolve({ stream: stream() }); + } + + if (command instanceof ConverseCommand) { + return Promise.resolve({ + metrics: { + latencyMs: 123, + }, + output: { + message: { + content: [{ text: "OK" }], + role: "assistant", + }, + }, + stopReason: "end_turn", + usage: { + inputTokens: 5, + outputTokens: 2, + totalTokens: 7, + }, + }); + } + + return Promise.resolve({ ignored: true }); + }, + ); + + const wrapped = wrapBedrockRuntime({ + send, + destroy() { + return this; + }, + }); + + expect(wrapped.destroy()).toBe(wrapped); + + await wrapped.send( + new ConverseCommand({ + inferenceConfig: { + maxTokens: 12, + temperature: 0, + }, + messages: [ + { + content: [{ text: "Reply with exactly OK." }], + role: "user", + }, + ], + modelId: "us.amazon.nova-lite-v1:0", + }), + ); + + const response = await wrapped.send( + new ConverseStreamCommand({ + messages: [ + { + content: [{ text: "Reply with exactly STREAM." }], + role: "user", + }, + ], + modelId: "us.amazon.nova-lite-v1:0", + }), + ); + for await (const _chunk of response.stream) { + // Consume the stream so chunk aggregation runs. + } + + await wrapped.send(new ListFoundationModelsCommand({})); + wrapped.send(new ConverseCommand({}), () => {}); + + const spans = await backgroundLogger.drain(); + expect(spans).toHaveLength(2); + + const converseSpan = spans.find( + (span: any) => span.span_attributes?.name === "bedrock.converse", + ) as Record | undefined; + const streamSpan = spans.find( + (span: any) => span.span_attributes?.name === "bedrock.converseStream", + ) as Record | undefined; + + expect(converseSpan?.metadata).toMatchObject({ + command: "ConverseCommand", + maxTokens: 12, + model: "us.amazon.nova-lite-v1:0", + operation: "converse", + provider: "aws-bedrock", + stopReason: "end_turn", + temperature: 0, + }); + expect(converseSpan?.input).toEqual({ + messages: [ + { + content: [{ text: "Reply with exactly OK." }], + role: "user", + }, + ], + system: undefined, + }); + expect(converseSpan?.output).toEqual({ + content: [{ text: "OK" }], + role: "assistant", + }); + expect(converseSpan?.metrics).toMatchObject({ + completion_tokens: 2, + latency_ms: 123, + prompt_tokens: 5, + tokens: 7, + }); + + expect(streamSpan?.metadata).toMatchObject({ + command: "ConverseStreamCommand", + model: "us.amazon.nova-lite-v1:0", + operation: "converseStream", + provider: "aws-bedrock", + stopReason: "end_turn", + }); + expect(streamSpan?.output).toEqual({ + content: [{ text: "STREAM" }], + role: "assistant", + }); + expect(streamSpan?.metrics).toMatchObject({ + completion_tokens: 1, + latency_ms: 42, + prompt_tokens: 4, + time_to_first_token: expect.any(Number), + tokens: 5, + }); + }); +}); diff --git a/js/src/wrappers/bedrock-runtime.ts b/js/src/wrappers/bedrock-runtime.ts new file mode 100644 index 000000000..f81f15bb6 --- /dev/null +++ b/js/src/wrappers/bedrock-runtime.ts @@ -0,0 +1,110 @@ +import { runWithAutoInstrumentationSuppressed } from "../instrumentation/auto-instrumentation-suppression"; +import { bedrockRuntimeChannels } from "../instrumentation/plugins/bedrock-runtime-channels"; +import { + buildBedrockRuntimeSpanInfo, + getBedrockRuntimeOperation, +} from "../instrumentation/plugins/bedrock-runtime-common"; +import type { + BedrockRuntimeClient, + BedrockRuntimeCommandLike, +} from "../vendor-sdk-types/bedrock-runtime"; + +/** + * Wrap an AWS Bedrock Runtime client with Braintrust tracing. + */ +export function wrapBedrockRuntime(client: T): T { + if (isSupportedBedrockRuntimeClient(client)) { + return bedrockRuntimeProxy(client) as T; + } + + // eslint-disable-next-line no-restricted-properties -- preserving wrapper warning behavior. + console.warn("Unsupported Bedrock Runtime library. Not wrapping."); + return client; +} + +const bedrockRuntimeProxyCache = new WeakMap< + BedrockRuntimeClient, + BedrockRuntimeClient +>(); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function isSupportedBedrockRuntimeClient( + value: unknown, +): value is BedrockRuntimeClient { + return isRecord(value) && typeof value.send === "function"; +} + +function bedrockRuntimeProxy( + client: BedrockRuntimeClient, +): BedrockRuntimeClient { + const cached = bedrockRuntimeProxyCache.get(client); + if (cached) { + return cached; + } + + const privateMethodWorkaroundCache = new WeakMap< + (...args: unknown[]) => unknown, + (...args: unknown[]) => unknown + >(); + + const proxy: BedrockRuntimeClient = new Proxy(client, { + get(target, prop, receiver) { + if (prop === "send") { + return wrapSend(target.send.bind(target)); + } + + const value = Reflect.get(target, prop, receiver); + if (typeof value !== "function") { + return value; + } + + const cachedValue = privateMethodWorkaroundCache.get(value); + if (cachedValue) { + return cachedValue; + } + + const thisBoundValue = function ( + this: unknown, + ...args: unknown[] + ): unknown { + const thisArg = this === proxy ? target : this; + const output = Reflect.apply(value, thisArg, args); + return output === target ? proxy : output; + }; + + privateMethodWorkaroundCache.set(value, thisBoundValue); + return thisBoundValue; + }, + }); + + bedrockRuntimeProxyCache.set(client, proxy); + return proxy; +} + +function wrapSend( + send: BedrockRuntimeClient["send"], +): BedrockRuntimeClient["send"] { + return (command, optionsOrCb, cb) => { + if ( + getBedrockRuntimeOperation(command) === undefined || + typeof optionsOrCb === "function" || + typeof cb === "function" + ) { + return send(command, optionsOrCb, cb); + } + + return bedrockRuntimeChannels.clientSend.tracePromise( + () => + runWithAutoInstrumentationSuppressed(() => + send(command, optionsOrCb), + ) as Promise, + { + arguments: [command as BedrockRuntimeCommandLike, optionsOrCb], + span_info: buildBedrockRuntimeSpanInfo(command), + }, + ); + }; +} From 3379db05e3e3f3fd25633720135ce4bacd44aa6b Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 19 Jun 2026 15:13:15 +0200 Subject: [PATCH 2/4] fix ci --- .../plugins/bedrock-runtime-plugin.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts b/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts index 175a1da6e..3141e37d8 100644 --- a/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts +++ b/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts @@ -272,7 +272,11 @@ function patchBedrockRuntimeStreamingResult(args: { } if (operation === "converseStream" && isAsyncIterable(args.result.stream)) { - patchConverseStream(args.result.stream, args.span, args.startTime); + patchConverseStream( + args.result.stream as AsyncIterable, + args.span, + args.startTime, + ); return true; } @@ -280,7 +284,11 @@ function patchBedrockRuntimeStreamingResult(args: { operation === "invokeModelWithResponseStream" && isAsyncIterable(args.result.body) ) { - patchInvokeModelResponseStream(args.result.body, args.span, args.startTime); + patchInvokeModelResponseStream( + args.result.body as AsyncIterable, + args.span, + args.startTime, + ); return true; } @@ -440,11 +448,12 @@ function aggregateInvokeModelResponseStreamChunks( const parsedChunks = chunks .map((chunk) => parseJsonBody(chunk.chunk?.bytes)) .filter((chunk) => chunk !== undefined); + const jsonLikeChunks = parsedChunks.filter(isObject); const text = parsedChunks.map(extractTextFromJsonLike).join(""); - const lastMetadataChunk = parsedChunks + const lastMetadataChunk = jsonLikeChunks .slice() .reverse() - .find((chunk) => isObject(chunk?.metadata)); + .find((chunk) => isObject(chunk.metadata)); const metadata = isObject(lastMetadataChunk?.metadata) ? sanitizeRecord(lastMetadataChunk.metadata) : undefined; @@ -455,7 +464,7 @@ function aggregateInvokeModelResponseStreamChunks( ? { text } : { chunk_count: chunks.length, - chunks: sanitizeBedrockValue(parsedChunks.slice(0, 20)), + chunks: sanitizeBedrockValue(jsonLikeChunks.slice(0, 20)), }, metrics: parseBedrockRuntimeMetrics( isObject(metadata) ? metadata.usage : undefined, From 5ba3e5be44bb7ed113aa9fe5f3e2d154d494c63c Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 19 Jun 2026 16:18:54 +0200 Subject: [PATCH 3/4] cs --- .changeset/salty-insects-jam.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/salty-insects-jam.md diff --git a/.changeset/salty-insects-jam.md b/.changeset/salty-insects-jam.md new file mode 100644 index 000000000..b5f73d5da --- /dev/null +++ b/.changeset/salty-insects-jam.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Add support for `@aws-sdk/client-bedrock-runtime` From 87b534b720ee3bc4ede2e2551f2370015c4ca9cb Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 19 Jun 2026 16:42:39 +0200 Subject: [PATCH 4/4] pp --- .../plugins/bedrock-runtime-plugin.test.ts | 84 +++++++++++++++++++ .../plugins/bedrock-runtime-plugin.ts | 14 +++- 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts b/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts index 3b642163b..7c72b31c7 100644 --- a/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts +++ b/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts @@ -206,4 +206,88 @@ describe("aggregateBedrockConverseStreamChunks", () => { }, }); }); + + it("drops prototype pollution keys from sanitized Bedrock objects", () => { + const delta = JSON.parse(`{ + "__proto__": { "pollutedFromDelta": true }, + "constructor": { "prototype": { "pollutedFromConstructor": true } }, + "extra": { + "__proto__": { "pollutedFromNestedDelta": true }, + "keep": true + }, + "prototype": { "pollutedFromPrototype": true }, + "text": "OK" + }`); + const maliciousAdditionalModelResponseFields = JSON.parse(`{ + "__proto__": { "pollutedFromMetadata": true }, + "constructor": { "prototype": { "pollutedFromConstructor": true } }, + "prototype": { "pollutedFromPrototype": true }, + "safe": { + "__proto__": { "pollutedFromNestedMetadata": true }, + "keep": true + } + }`); + + expect(Object.getOwnPropertyDescriptor(delta, "__proto__")).toBeDefined(); + expect( + Object.getOwnPropertyDescriptor( + maliciousAdditionalModelResponseFields, + "__proto__", + ), + ).toBeDefined(); + + const result = aggregateBedrockConverseStreamChunks([ + { + contentBlockDelta: { + contentBlockIndex: 0, + delta, + }, + }, + { + messageStop: { + additionalModelResponseFields: maliciousAdditionalModelResponseFields, + }, + }, + ]); + + const content = ( + result.output as { content: Array> } + ).content[0]; + const extra = content.extra as Record; + const metadata = result.metadata as Record; + const sanitizedAdditionalModelResponseFields = + metadata.additionalModelResponseFields as Record; + const safe = sanitizedAdditionalModelResponseFields.safe as Record< + string, + unknown + >; + + expect(content.text).toBe("OK"); + expect(extra.keep).toBe(true); + expect(safe.keep).toBe(true); + + for (const value of [ + content, + extra, + sanitizedAdditionalModelResponseFields, + safe, + {}, + ]) { + expect("pollutedFromDelta" in value).toBe(false); + expect("pollutedFromMetadata" in value).toBe(false); + expect("pollutedFromNestedDelta" in value).toBe(false); + expect("pollutedFromNestedMetadata" in value).toBe(false); + expect("pollutedFromConstructor" in value).toBe(false); + expect("pollutedFromPrototype" in value).toBe(false); + expect(Object.getOwnPropertyDescriptor(value, "__proto__")).toBe( + undefined, + ); + expect(Object.getOwnPropertyDescriptor(value, "constructor")).toBe( + undefined, + ); + expect(Object.getOwnPropertyDescriptor(value, "prototype")).toBe( + undefined, + ); + } + }); }); diff --git a/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts b/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts index 3141e37d8..49860d925 100644 --- a/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts +++ b/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts @@ -510,7 +510,12 @@ function mergeContentBlockDelta( } for (const [key, value] of Object.entries(delta)) { - if (key !== "text" && key !== "reasoningContent" && key !== "toolUse") { + if ( + key !== "text" && + key !== "reasoningContent" && + key !== "toolUse" && + isSafeBedrockObjectKey(key) + ) { next[key] = sanitizeBedrockValue(value); } } @@ -590,11 +595,18 @@ function sanitizeBedrockValue(value: unknown, depth = 0): unknown { const output: Record = {}; for (const [key, nested] of Object.entries(value)) { + if (!isSafeBedrockObjectKey(key)) { + continue; + } output[key] = sanitizeBedrockValue(nested, depth + 1); } return output; } +function isSafeBedrockObjectKey(key: string): boolean { + return key !== "__proto__" && key !== "constructor" && key !== "prototype"; +} + function extractTextFromJsonLike(value: unknown): string { if (typeof value === "string") { return value;