test: Add e2e tests for @anthropic-ai/vertex-sdk

.changeset/anthropic-vertex-instrumentation.md

@@ -0,0 +1,5 @@
+---
+"braintrust": patch
+---
+
+test: Add e2e tests for `@anthropic-ai/vertex-sdk`

e2e/config/pr-comment-scenarios.json

@@ -408,8 +408,14 @@
     "label": "LangSmith Instrumentation",
     "metadataScenario": "langsmith-instrumentation",
     "variants": [
-      { "variantKey": "langsmith-v0", "label": "v0 pinned" },
-      { "variantKey": "langsmith-v0-latest", "label": "v0 latest" }
+      {
+        "variantKey": "langsmith-v0",
+        "label": "v0 pinned"
+      },
+      {
+        "variantKey": "langsmith-v0-latest",
+        "label": "v0 latest"
+      }
     ]
   },
   {
@@ -654,10 +660,22 @@
     "label": "Eve Instrumentation",
     "metadataScenario": "eve-instrumentation",
     "variants": [
-      { "variantKey": "eve-v0", "label": "v0 pinned" },
-      { "variantKey": "eve-v0-provider", "label": "v0 provider minimum" },
-      { "variantKey": "eve-v0-latest-pinned", "label": "v0 latest pinned" },
-      { "variantKey": "eve-v0-latest", "label": "v0 latest" }
+      {
+        "variantKey": "eve-v0",
+        "label": "v0 pinned"
+      },
+      {
+        "variantKey": "eve-v0-provider",
+        "label": "v0 provider minimum"
+      },
+      {
+        "variantKey": "eve-v0-latest-pinned",
+        "label": "v0 latest pinned"
+      },
+      {
+        "variantKey": "eve-v0-latest",
+        "label": "v0 latest"
+      }
     ]
   },
   {
@@ -778,6 +796,21 @@
       }
     ]
   },
+  {
+    "scenarioDirName": "anthropic-vertex-instrumentation",
+    "label": "Anthropic Vertex Instrumentation",
+    "metadataScenario": "anthropic-vertex-instrumentation",
+    "variants": [
+      {
+        "variantKey": "anthropic-vertex-v0",
+        "label": "v0 pinned"
+      },
+      {
+        "variantKey": "anthropic-vertex-v0-latest",
+        "label": "v0 latest"
+      }
+    ]
+  },
   {
     "scenarioDirName": "elevenlabs-instrumentation",
     "label": "ElevenLabs Instrumentation",

e2e/helpers/scenario-harness.ts

@@ -301,6 +301,13 @@ function getCassetteServerRoutes(): CassetteServerRoute[] {
       prefix: "/aws-bedrock-runtime",
       upstreamOrigin: `https://bedrock-runtime.${getBedrockRegion()}.amazonaws.com`,
     },
+    {
+      prefix: "/anthropic-vertex",
+      upstreamOrigin:
+        process.env.CLOUD_ML_REGION && process.env.CLOUD_ML_REGION !== "global"
+          ? `https://${process.env.CLOUD_ML_REGION}-aiplatform.googleapis.com`
+          : "https://aiplatform.googleapis.com",
+    },
     { prefix: "/elevenlabs", upstreamOrigin: "https://api.elevenlabs.io" },
     { prefix: "/cohere", upstreamOrigin: "https://api.cohere.com" },
     { prefix: "/cursor/v1", upstreamOrigin: "https://api.cursor.com/v1" },
@@ -336,6 +343,7 @@ function getCassetteEnv(wiring: ActiveCassetteWiring): Record<string, string> {
     BRAINTRUST_E2E_MODEL_BASE_URL: `${serverUrl}/openai/v1`,
     ANTHROPIC_BASE_URL: `${serverUrl}/anthropic`,
     ANTHROPIC_BEDROCK_BASE_URL: `${serverUrl}/anthropic-bedrock`,
+    ANTHROPIC_VERTEX_BASE_URL: `${serverUrl}/anthropic-vertex/v1`,
     AWS_BEDROCK_RUNTIME_BASE_URL: `${serverUrl}/aws-bedrock-runtime`,
     ELEVENLABS_BASE_URL: `${serverUrl}/elevenlabs`,
     COHERE_BASE_URL: `${serverUrl}/cohere`,

e2e/scenarios/anthropic-vertex-instrumentation/assertions.ts

@@ -0,0 +1,79 @@
+import { beforeAll, describe, expect, test } from "vitest";
+import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server";
+import { resolveFileSnapshotPath } from "../../helpers/file-snapshot";
+import { withScenarioHarness } from "../../helpers/scenario-harness";
+import { matchSpanTreeSnapshot } from "../../helpers/span-tree";
+import {
+  findAllSpans,
+  findChildSpans,
+  findLatestSpan,
+} from "../../helpers/trace-selectors";
+
+const ROOT_NAME = "anthropic-vertex-instrumentation-root";
+
+export function defineAnthropicVertexInstrumentationAssertions(options: {
+  name: string;
+  runScenario: Parameters<typeof withScenarioHarness>[0];
+  snapshotName: string;
+  testFileUrl: string;
+  timeoutMs: number;
+}): void {
+  describe(options.name, () => {
+    let events: CapturedLogEvent[] = [];
+    beforeAll(async () => {
+      await withScenarioHarness(async (harness) => {
+        await options.runScenario(harness);
+        events = harness.events();
+      });
+    }, options.timeoutMs);
+
+    test("captures one Anthropic span per Vertex call", () => {
+      const root = findLatestSpan(events, ROOT_NAME);
+      expect(root).toBeDefined();
+      const llmSpans = findAllSpans(events, "anthropic.messages.create");
+      expect(llmSpans).toHaveLength(6);
+      for (const beta of [false, true]) {
+        for (const operation of ["create", "stream", "stream-helper"]) {
+          const parent = findLatestSpan(
+            events,
+            `anthropic-vertex-${beta ? "beta-" : ""}${operation}-operation`,
+          );
+          expect(parent?.span.parentIds).toEqual([root?.span.id]);
+          const spans = findChildSpans(
+            events,
+            "anthropic.messages.create",
+            parent?.span.id,
+          );
+          expect(spans).toHaveLength(1);
+          const span = spans[0]!;
+          expect(span.row.metadata).toMatchObject({
+            provider: "anthropic",
+            model: expect.any(String),
+          });
+          expect(span.input).toBeDefined();
+          expect(span.output).toBeDefined();
+          expect(span.metrics).toMatchObject({
+            prompt_tokens: expect.any(Number),
+            completion_tokens: expect.any(Number),
+          });
+          if (operation !== "create") {
+            expect(span.metrics?.time_to_first_token).toEqual(
+              expect.any(Number),
+            );
+          }
+          expect(span.row.error).toBeUndefined();
+        }
+      }
+    });
+
+    test("matches span tree snapshot", async () => {
+      await matchSpanTreeSnapshot(
+        events,
+        resolveFileSnapshotPath(
+          options.testFileUrl,
+          `${options.snapshotName}.span-tree.json`,
+        ),
+      );
+    });
+  });
+}

e2e/scenarios/anthropic-vertex-instrumentation/cassette-filter.mjs

@@ -0,0 +1,15 @@
+// @ts-check
+/** @type {import("@braintrust/seinfeld").FilterSpec} */
+export const filter = [
+  "default",
+  {
+    normalizeRequest(req) {
+      const url = new URL(req.url);
+      url.pathname = url.pathname.replace(
+        /\/projects\/[^/]+\//,
+        "/projects/cassette-project/",
+      );
+      return { ...req, url: url.toString() };
+    },
+  },
+];

e2e/scenarios/anthropic-vertex-instrumentation/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "@braintrust/e2e-anthropic-vertex-instrumentation",
+  "private": true,
+  "braintrustScenario": {
+    "bump": {
+      "dependencies": {
+        "anthropic-vertex-sdk-v0-latest": {
+          "package": "@anthropic-ai/vertex-sdk",
+          "range": "0"
+        }
+      }
+    }
+  },
+  "dependencies": {
+    "anthropic-vertex-sdk-v0": "npm:@anthropic-ai/vertex-sdk@0.14.3",
+    "anthropic-vertex-sdk-v0-latest": "npm:@anthropic-ai/vertex-sdk@0.19.7",
+    "google-auth-library": "10.2.0"
+  }
+}