diff --git a/.changeset/anthropic-vertex-instrumentation.md b/.changeset/anthropic-vertex-instrumentation.md new file mode 100644 index 000000000..c1b805664 --- /dev/null +++ b/.changeset/anthropic-vertex-instrumentation.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +test: Add e2e tests for `@anthropic-ai/vertex-sdk` diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index 696cf0c64..ba9add43b 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -408,8 +408,14 @@ "label": "LangSmith Instrumentation", "metadataScenario": "langsmith-instrumentation", "variants": [ - { "variantKey": "langsmith-v0", "label": "v0 pinned" }, - { "variantKey": "langsmith-v0-latest", "label": "v0 latest" } + { + "variantKey": "langsmith-v0", + "label": "v0 pinned" + }, + { + "variantKey": "langsmith-v0-latest", + "label": "v0 latest" + } ] }, { @@ -654,10 +660,22 @@ "label": "Eve Instrumentation", "metadataScenario": "eve-instrumentation", "variants": [ - { "variantKey": "eve-v0", "label": "v0 pinned" }, - { "variantKey": "eve-v0-provider", "label": "v0 provider minimum" }, - { "variantKey": "eve-v0-latest-pinned", "label": "v0 latest pinned" }, - { "variantKey": "eve-v0-latest", "label": "v0 latest" } + { + "variantKey": "eve-v0", + "label": "v0 pinned" + }, + { + "variantKey": "eve-v0-provider", + "label": "v0 provider minimum" + }, + { + "variantKey": "eve-v0-latest-pinned", + "label": "v0 latest pinned" + }, + { + "variantKey": "eve-v0-latest", + "label": "v0 latest" + } ] }, { @@ -778,6 +796,21 @@ } ] }, + { + "scenarioDirName": "anthropic-vertex-instrumentation", + "label": "Anthropic Vertex Instrumentation", + "metadataScenario": "anthropic-vertex-instrumentation", + "variants": [ + { + "variantKey": "anthropic-vertex-v0", + "label": "v0 pinned" + }, + { + "variantKey": "anthropic-vertex-v0-latest", + "label": "v0 latest" + } + ] + }, { "scenarioDirName": "elevenlabs-instrumentation", "label": "ElevenLabs Instrumentation", diff --git a/e2e/helpers/scenario-harness.ts b/e2e/helpers/scenario-harness.ts index bd90a6dec..ac75a8f22 100644 --- a/e2e/helpers/scenario-harness.ts +++ b/e2e/helpers/scenario-harness.ts @@ -301,6 +301,13 @@ function getCassetteServerRoutes(): CassetteServerRoute[] { prefix: "/aws-bedrock-runtime", upstreamOrigin: `https://bedrock-runtime.${getBedrockRegion()}.amazonaws.com`, }, + { + prefix: "/anthropic-vertex", + upstreamOrigin: + process.env.CLOUD_ML_REGION && process.env.CLOUD_ML_REGION !== "global" + ? `https://${process.env.CLOUD_ML_REGION}-aiplatform.googleapis.com` + : "https://aiplatform.googleapis.com", + }, { prefix: "/elevenlabs", upstreamOrigin: "https://api.elevenlabs.io" }, { prefix: "/cohere", upstreamOrigin: "https://api.cohere.com" }, { prefix: "/cursor/v1", upstreamOrigin: "https://api.cursor.com/v1" }, @@ -336,6 +343,7 @@ function getCassetteEnv(wiring: ActiveCassetteWiring): Record { BRAINTRUST_E2E_MODEL_BASE_URL: `${serverUrl}/openai/v1`, ANTHROPIC_BASE_URL: `${serverUrl}/anthropic`, ANTHROPIC_BEDROCK_BASE_URL: `${serverUrl}/anthropic-bedrock`, + ANTHROPIC_VERTEX_BASE_URL: `${serverUrl}/anthropic-vertex/v1`, AWS_BEDROCK_RUNTIME_BASE_URL: `${serverUrl}/aws-bedrock-runtime`, ELEVENLABS_BASE_URL: `${serverUrl}/elevenlabs`, COHERE_BASE_URL: `${serverUrl}/cohere`, diff --git a/e2e/scenarios/anthropic-vertex-instrumentation/assertions.ts b/e2e/scenarios/anthropic-vertex-instrumentation/assertions.ts new file mode 100644 index 000000000..d42504c3c --- /dev/null +++ b/e2e/scenarios/anthropic-vertex-instrumentation/assertions.ts @@ -0,0 +1,79 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; +import { withScenarioHarness } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; +import { + findAllSpans, + findChildSpans, + findLatestSpan, +} from "../../helpers/trace-selectors"; + +const ROOT_NAME = "anthropic-vertex-instrumentation-root"; + +export function defineAnthropicVertexInstrumentationAssertions(options: { + name: string; + runScenario: Parameters[0]; + snapshotName: string; + testFileUrl: string; + timeoutMs: number; +}): void { + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + }); + }, options.timeoutMs); + + test("captures one Anthropic span per Vertex call", () => { + const root = findLatestSpan(events, ROOT_NAME); + expect(root).toBeDefined(); + const llmSpans = findAllSpans(events, "anthropic.messages.create"); + expect(llmSpans).toHaveLength(6); + for (const beta of [false, true]) { + for (const operation of ["create", "stream", "stream-helper"]) { + const parent = findLatestSpan( + events, + `anthropic-vertex-${beta ? "beta-" : ""}${operation}-operation`, + ); + expect(parent?.span.parentIds).toEqual([root?.span.id]); + const spans = findChildSpans( + events, + "anthropic.messages.create", + parent?.span.id, + ); + expect(spans).toHaveLength(1); + const span = spans[0]!; + expect(span.row.metadata).toMatchObject({ + provider: "anthropic", + model: expect.any(String), + }); + expect(span.input).toBeDefined(); + expect(span.output).toBeDefined(); + expect(span.metrics).toMatchObject({ + prompt_tokens: expect.any(Number), + completion_tokens: expect.any(Number), + }); + if (operation !== "create") { + expect(span.metrics?.time_to_first_token).toEqual( + expect.any(Number), + ); + } + expect(span.row.error).toBeUndefined(); + } + } + }); + + test("matches span tree snapshot", async () => { + await matchSpanTreeSnapshot( + events, + resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-tree.json`, + ), + ); + }); + }); +} diff --git a/e2e/scenarios/anthropic-vertex-instrumentation/cassette-filter.mjs b/e2e/scenarios/anthropic-vertex-instrumentation/cassette-filter.mjs new file mode 100644 index 000000000..3edb16b3b --- /dev/null +++ b/e2e/scenarios/anthropic-vertex-instrumentation/cassette-filter.mjs @@ -0,0 +1,15 @@ +// @ts-check +/** @type {import("@braintrust/seinfeld").FilterSpec} */ +export const filter = [ + "default", + { + normalizeRequest(req) { + const url = new URL(req.url); + url.pathname = url.pathname.replace( + /\/projects\/[^/]+\//, + "/projects/cassette-project/", + ); + return { ...req, url: url.toString() }; + }, + }, +]; diff --git a/e2e/scenarios/anthropic-vertex-instrumentation/package.json b/e2e/scenarios/anthropic-vertex-instrumentation/package.json new file mode 100644 index 000000000..e25444e74 --- /dev/null +++ b/e2e/scenarios/anthropic-vertex-instrumentation/package.json @@ -0,0 +1,19 @@ +{ + "name": "@braintrust/e2e-anthropic-vertex-instrumentation", + "private": true, + "braintrustScenario": { + "bump": { + "dependencies": { + "anthropic-vertex-sdk-v0-latest": { + "package": "@anthropic-ai/vertex-sdk", + "range": "0" + } + } + } + }, + "dependencies": { + "anthropic-vertex-sdk-v0": "npm:@anthropic-ai/vertex-sdk@0.14.3", + "anthropic-vertex-sdk-v0-latest": "npm:@anthropic-ai/vertex-sdk@0.19.7", + "google-auth-library": "10.2.0" + } +} diff --git a/e2e/scenarios/anthropic-vertex-instrumentation/pnpm-lock.yaml b/e2e/scenarios/anthropic-vertex-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..dbdcb2f4d --- /dev/null +++ b/e2e/scenarios/anthropic-vertex-instrumentation/pnpm-lock.yaml @@ -0,0 +1,398 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + anthropic-vertex-sdk-v0: + specifier: npm:@anthropic-ai/vertex-sdk@0.14.3 + version: '@anthropic-ai/vertex-sdk@0.14.3' + anthropic-vertex-sdk-v0-latest: + specifier: npm:@anthropic-ai/vertex-sdk@0.19.7 + version: '@anthropic-ai/vertex-sdk@0.19.7' + google-auth-library: + specifier: 10.2.0 + version: 10.2.0 + +packages: + + '@anthropic-ai/sdk@0.124.0': + resolution: {integrity: sha512-cN5O8i9UVxHeOQAzj/XjshWXG8KiibJDw9OGpH2Z/eR3n/RBxdoLxDJOcfqAJWvjaMDFfHTBADU04hWRJVkDyA==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + + '@anthropic-ai/vertex-sdk@0.14.3': + resolution: {integrity: sha512-GJZTkkvN66gM3Epqm9laKEjC3orQqzmQt8JAgTN9+zlb+I+1/oEd3Z7rj2tkEKCTeOUVScdhcXPudN8GdpuGqA==} + + '@anthropic-ai/vertex-sdk@0.19.7': + resolution: {integrity: sha512-WbSYU84IAGPPCz02se6tJNc2X4Ij1itcj5+9L27boYCYDOwPLiYT5/bVmcygCy9pD6fn3Hc/8/WtuJBBQUAP2w==} + + '@babel/runtime@7.29.7': + resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} + engines: {node: '>=6.9.0'} + + '@stablelib/base64@1.0.1': + resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==} + + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + bignumber.js@9.3.1: + resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} + + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + + extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + + fast-sha256@1.3.0: + resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==} + + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + + gaxios@6.7.1: + resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} + engines: {node: '>=14'} + + gaxios@7.3.1: + resolution: {integrity: sha512-kB3rzJV7d9juLZh8/56QTXCwQfxyhdOMdyYk1HdQKFtF8TJTDTZQJtixWIwXdE9Jji91mC41DUNpjleo4L4eAQ==} + engines: {node: '>=18'} + + gcp-metadata@6.1.1: + resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==} + engines: {node: '>=14'} + + gcp-metadata@7.0.1: + resolution: {integrity: sha512-UcO3kefx6dCcZkgcTGgVOTFb7b1LlQ02hY1omMjjrrBzkajRMCFgYOjs7J71WqnuG1k2b+9ppGL7FsOfhZMQKQ==} + engines: {node: '>=18'} + + google-auth-library@10.2.0: + resolution: {integrity: sha512-gy/0hRx8+Ye0HlUm3GrfpR4lbmJQ6bJ7F44DmN7GtMxxzWSojLzx0Bhv/hj7Wlj7a2On0FcT8jrz8Y1c1nxCyg==} + engines: {node: '>=18'} + + google-auth-library@9.15.1: + resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} + engines: {node: '>=14'} + + google-logging-utils@0.0.2: + resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==} + engines: {node: '>=14'} + + google-logging-utils@1.2.0: + resolution: {integrity: sha512-WE9av4wKDZgRjBwgVUabocx8T6/7o3Ca1Fat46FXDhXVAFibzNadedcOXrdgd1Kzmk8tsk/9ZH89Wyf/SqeZ3A==} + engines: {node: '>=18'} + + gtoken@7.1.0: + resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} + engines: {node: '>=14.0.0'} + + gtoken@8.0.0: + resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==} + engines: {node: '>=18'} + + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + + is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + + json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + + json-schema-to-ts@3.1.1: + resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==} + engines: {node: '>=16'} + + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + standardwebhooks@1.1.1: + resolution: {integrity: sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ==} + + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + + ts-algebra@2.0.0: + resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} + + uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). + hasBin: true + + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + +snapshots: + + '@anthropic-ai/sdk@0.124.0': + dependencies: + json-schema-to-ts: 3.1.1 + standardwebhooks: 1.1.1 + + '@anthropic-ai/vertex-sdk@0.14.3': + dependencies: + '@anthropic-ai/sdk': 0.124.0 + google-auth-library: 9.15.1 + transitivePeerDependencies: + - encoding + - supports-color + - zod + + '@anthropic-ai/vertex-sdk@0.19.7': + dependencies: + '@anthropic-ai/sdk': 0.124.0 + google-auth-library: 10.2.0 + transitivePeerDependencies: + - supports-color + - zod + + '@babel/runtime@7.29.7': {} + + '@stablelib/base64@1.0.1': {} + + agent-base@7.1.4: {} + + base64-js@1.5.1: {} + + bignumber.js@9.3.1: {} + + buffer-equal-constant-time@1.0.1: {} + + data-uri-to-buffer@4.0.1: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.2.1 + + extend@3.0.2: {} + + fast-sha256@1.3.0: {} + + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + + gaxios@6.7.1: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + is-stream: 2.0.1 + node-fetch: 2.7.0 + uuid: 9.0.1 + transitivePeerDependencies: + - encoding + - supports-color + + gaxios@7.3.1: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + transitivePeerDependencies: + - supports-color + + gcp-metadata@6.1.1: + dependencies: + gaxios: 6.7.1 + google-logging-utils: 0.0.2 + json-bigint: 1.0.0 + transitivePeerDependencies: + - encoding + - supports-color + + gcp-metadata@7.0.1: + dependencies: + gaxios: 7.3.1 + google-logging-utils: 1.2.0 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + + google-auth-library@10.2.0: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.3.1 + gcp-metadata: 7.0.1 + google-logging-utils: 1.2.0 + gtoken: 8.0.0 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + + google-auth-library@9.15.1: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 6.7.1 + gcp-metadata: 6.1.1 + gtoken: 7.1.0 + jws: 4.0.1 + transitivePeerDependencies: + - encoding + - supports-color + + google-logging-utils@0.0.2: {} + + google-logging-utils@1.2.0: {} + + gtoken@7.1.0: + dependencies: + gaxios: 6.7.1 + jws: 4.0.1 + transitivePeerDependencies: + - encoding + - supports-color + + gtoken@8.0.0: + dependencies: + gaxios: 7.3.1 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + is-stream@2.0.1: {} + + json-bigint@1.0.0: + dependencies: + bignumber.js: 9.3.1 + + json-schema-to-ts@3.1.1: + dependencies: + '@babel/runtime': 7.29.7 + ts-algebra: 2.0.0 + + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + + ms@2.1.3: {} + + node-domexception@1.0.0: {} + + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + + safe-buffer@5.2.1: {} + + standardwebhooks@1.1.1: + dependencies: + '@stablelib/base64': 1.0.1 + fast-sha256: 1.3.0 + + tr46@0.0.3: {} + + ts-algebra@2.0.0: {} + + uuid@9.0.1: {} + + web-streams-polyfill@3.3.3: {} + + webidl-conversions@3.0.1: {} + + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 diff --git a/e2e/scenarios/anthropic-vertex-instrumentation/scenario.cjs b/e2e/scenarios/anthropic-vertex-instrumentation/scenario.cjs new file mode 100644 index 000000000..e8e085aea --- /dev/null +++ b/e2e/scenarios/anthropic-vertex-instrumentation/scenario.cjs @@ -0,0 +1,19 @@ +const packageName = + process.env.ANTHROPIC_VERTEX_PACKAGE_NAME || "anthropic-vertex-sdk-v0-latest"; +const mod = require(packageName); +const AnthropicVertex = mod.default ?? mod.AnthropicVertex ?? mod; + +void (async () => { + const { runMain } = await import("../../helpers/provider-runtime.mjs"); + const { runAutoAnthropicVertexInstrumentation } = + await import("./scenario.impl.mjs"); + + runMain(async () => { + await runAutoAnthropicVertexInstrumentation({ + AnthropicVertex, + }); + }); +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/e2e/scenarios/anthropic-vertex-instrumentation/scenario.impl.mjs b/e2e/scenarios/anthropic-vertex-instrumentation/scenario.impl.mjs new file mode 100644 index 000000000..9caca4204 --- /dev/null +++ b/e2e/scenarios/anthropic-vertex-instrumentation/scenario.impl.mjs @@ -0,0 +1,104 @@ +import assert from "node:assert/strict"; +import { OAuth2Client } from "google-auth-library"; +import { wrapAnthropic } from "braintrust"; +import { + collectAsync, + runOperation, + runTracedScenario, +} from "../../helpers/provider-runtime.mjs"; + +export const ROOT_NAME = "anthropic-vertex-instrumentation-root"; +export const SCENARIO_NAME = "anthropic-vertex-instrumentation"; + +async function runAnthropicVertexInstrumentationScenario(options) { + const replay = process.env.BRAINTRUST_E2E_CASSETTE_MODE === "replay"; + const authClient = new OAuth2Client(); + authClient.setCredentials({ + access_token: replay ? "cassette-placeholder" : process.env.VERTEX_API_KEY, + }); + const baseClient = new options.AnthropicVertex({ + authClient, + projectId: replay + ? "cassette-project" + : (process.env.VERTEX_PROJECT_ID ?? + process.env.ANTHROPIC_VERTEX_PROJECT_ID), + region: process.env.CLOUD_ML_REGION || "global", + baseURL: process.env.ANTHROPIC_VERTEX_BASE_URL, + maxRetries: 0, + }); + const client = options.decorateClient + ? options.decorateClient(baseClient) + : baseClient; + const model = "claude-sonnet-4-5@20250929"; + + await runTracedScenario({ + callback: async () => { + for (const beta of [false, true]) { + const messages = beta ? client.beta.messages : client.messages; + const prefix = beta ? "beta-" : ""; + await runOperation( + `anthropic-vertex-${prefix}create-operation`, + "create", + async () => { + const result = await messages + .create({ + model, + max_tokens: 32, + messages: [{ role: "user", content: "Reply with exactly OK." }], + temperature: 0, + }) + .withResponse(); + assert.equal(result.response.status, 200); + assert.equal(result.data.role, "assistant"); + }, + ); + await runOperation( + `anthropic-vertex-${prefix}stream-operation`, + "stream", + async () => { + const stream = await messages.create({ + model, + max_tokens: 32, + messages: [{ role: "user", content: "Count from one to three." }], + stream: true, + temperature: 0, + }); + const events = await collectAsync(stream); + assert(events.some((event) => event.type === "message_stop")); + }, + ); + await runOperation( + `anthropic-vertex-${prefix}stream-helper-operation`, + "stream-helper", + async () => { + const stream = messages.stream({ + model, + max_tokens: 32, + messages: [ + { role: "user", content: "Reply with exactly Hello." }, + ], + temperature: 0, + }); + const result = await stream.finalMessage(); + assert.equal(result.role, "assistant"); + assert(result.content.length > 0); + }, + ); + } + }, + metadata: { scenario: SCENARIO_NAME }, + projectNameBase: "tmp-luca-e2e-anthropic-vertex-instrumentation", + rootName: ROOT_NAME, + }); +} + +export async function runWrappedAnthropicVertexInstrumentation(options) { + await runAnthropicVertexInstrumentationScenario({ + decorateClient: wrapAnthropic, + ...options, + }); +} + +export async function runAutoAnthropicVertexInstrumentation(options) { + await runAnthropicVertexInstrumentationScenario(options); +} diff --git a/e2e/scenarios/anthropic-vertex-instrumentation/scenario.mjs b/e2e/scenarios/anthropic-vertex-instrumentation/scenario.mjs new file mode 100644 index 000000000..44255778d --- /dev/null +++ b/e2e/scenarios/anthropic-vertex-instrumentation/scenario.mjs @@ -0,0 +1,11 @@ +const anthropicVertexPackageName = + process.env.ANTHROPIC_VERTEX_PACKAGE_NAME ?? "anthropic-vertex-sdk-v0-latest"; +const { default: AnthropicVertex } = await import(anthropicVertexPackageName); +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoAnthropicVertexInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => { + await runAutoAnthropicVertexInstrumentation({ + AnthropicVertex, + }); +}); diff --git a/e2e/scenarios/anthropic-vertex-instrumentation/scenario.test.ts b/e2e/scenarios/anthropic-vertex-instrumentation/scenario.test.ts new file mode 100644 index 000000000..bf53ec373 --- /dev/null +++ b/e2e/scenarios/anthropic-vertex-instrumentation/scenario.test.ts @@ -0,0 +1,102 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineAnthropicVertexInstrumentationAssertions } from "./assertions"; +const ANTHROPIC_VERTEX_SCENARIO_TIMEOUT_MS = 180_000; + +const originalScenarioDir = resolveScenarioDir(import.meta.url); +const scenarioDir = await prepareScenarioDir({ + scenarioDir: originalScenarioDir, +}); +const anthropicVertexScenarios = await Promise.all( + [ + { + dependencyName: "anthropic-vertex-sdk-v0", + snapshotName: "anthropic-vertex-v0", + }, + { + dependencyName: "anthropic-vertex-sdk-v0-latest", + snapshotName: "anthropic-vertex-v0-latest", + }, + ].map(async (scenario) => ({ + ...scenario, + version: await readInstalledPackageVersion( + scenarioDir, + scenario.dependencyName, + ), + })), +); + +describe.concurrent("variants", () => { + for (const scenario of anthropicVertexScenarios) { + describe.sequential(`anthropic vertex sdk ${scenario.version}`, () => { + defineAnthropicVertexInstrumentationAssertions({ + name: "wrapped instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: "scenario.ts", + env: { + ANTHROPIC_VERTEX_PACKAGE_NAME: scenario.dependencyName, + }, + runContext: { + variantKey: scenario.snapshotName, + originalScenarioDir, + }, + scenarioDir, + timeoutMs: ANTHROPIC_VERTEX_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: `${scenario.snapshotName}-wrapped`, + testFileUrl: import.meta.url, + timeoutMs: ANTHROPIC_VERTEX_SCENARIO_TIMEOUT_MS, + }); + + defineAnthropicVertexInstrumentationAssertions({ + name: "auto-hook instrumentation ESM", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: "scenario.mjs", + env: { + ANTHROPIC_VERTEX_PACKAGE_NAME: scenario.dependencyName, + }, + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext: { + variantKey: scenario.snapshotName, + originalScenarioDir, + }, + scenarioDir, + timeoutMs: ANTHROPIC_VERTEX_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: `${scenario.snapshotName}-auto-esm`, + testFileUrl: import.meta.url, + timeoutMs: ANTHROPIC_VERTEX_SCENARIO_TIMEOUT_MS, + }); + + defineAnthropicVertexInstrumentationAssertions({ + name: "auto-hook instrumentation CJS", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: "scenario.cjs", + env: { + ANTHROPIC_VERTEX_PACKAGE_NAME: scenario.dependencyName, + }, + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext: { + variantKey: scenario.snapshotName, + originalScenarioDir, + }, + scenarioDir, + timeoutMs: ANTHROPIC_VERTEX_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: `${scenario.snapshotName}-auto-cjs`, + testFileUrl: import.meta.url, + timeoutMs: ANTHROPIC_VERTEX_SCENARIO_TIMEOUT_MS, + }); + }); + } +}); diff --git a/e2e/scenarios/anthropic-vertex-instrumentation/scenario.ts b/e2e/scenarios/anthropic-vertex-instrumentation/scenario.ts new file mode 100644 index 000000000..720604c7d --- /dev/null +++ b/e2e/scenarios/anthropic-vertex-instrumentation/scenario.ts @@ -0,0 +1,11 @@ +const anthropicVertexPackageName = + process.env.ANTHROPIC_VERTEX_PACKAGE_NAME ?? "anthropic-vertex-sdk-v0-latest"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runWrappedAnthropicVertexInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => { + const { default: AnthropicVertex } = await import(anthropicVertexPackageName); + await runWrappedAnthropicVertexInstrumentation({ + AnthropicVertex, + }); +}); diff --git a/js/src/auto-instrumentations/configs/anthropic.ts b/js/src/auto-instrumentations/configs/anthropic.ts index 3acaa4d75..8f19353c9 100644 --- a/js/src/auto-instrumentations/configs/anthropic.ts +++ b/js/src/auto-instrumentations/configs/anthropic.ts @@ -14,9 +14,9 @@ import { anthropicChannels } from "../../instrumentation/plugins/anthropic-chann */ export const anthropicConfigs: InstrumentationConfig[] = [ // Each logical target is listed for both published module formats: - // `.mjs` covers ESM imports, while `.js` covers CJS requires. The Bedrock - // SDK delegates CJS `messages.create` calls through these Anthropic SDK - // `.js` resource files. + // `.mjs` covers ESM imports, while `.js` covers CJS requires. Both + // @anthropic-ai/bedrock-sdk and @anthropic-ai/vertex-sdk instantiate these + // shared Anthropic resources, so they use the same targets and channels. // Messages API - create in older SDK layouts (supports streaming via stream=true parameter) { diff --git a/js/src/wrappers/anthropic.ts b/js/src/wrappers/anthropic.ts index 12aae0218..38b206fd1 100644 --- a/js/src/wrappers/anthropic.ts +++ b/js/src/wrappers/anthropic.ts @@ -15,6 +15,9 @@ import type { * Wrap an `Anthropic` object (created with `new Anthropic(...)`) so calls emit * tracing-channel events that Braintrust plugins can consume. * + * Also supports `AnthropicBedrock` and `AnthropicVertex` clients through their + * shared Anthropic message resources. + * * Currently, this only supports the `v4` API. Sessions streams are eligible * for collection only after being passed to `collectAnthropicSession()`; * wrapping and consuming a Sessions stream alone does not emit spans.