diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c7b327..739a451 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -253,8 +253,13 @@ jobs: chmod +x packages/cli/typescript/dist-binary/browserstack-client-ts-linux-* chmod +x packages/cli/golang/dist/browserstack-client-linux-* + # Dogfooding: results stream live to BrowserStack Test Reporting & Analytics via our own SDK + # (see scripts/ci-report/tra-reporter.mjs); reporting errors never fail the job. - name: Test - run: pnpm test + env: + TRA_LABEL: linux + run: pnpm test --reporter=default --reporter=./scripts/ci-report/tra-reporter.mjs + # ── 2b. CLI binary tests on macOS and Windows ──────────────────────────────── test-cli: @@ -301,7 +306,16 @@ jobs: chmod +x packages/cli/golang/dist/browserstack-client-darwin-* - name: Test CLI - run: pnpm test -- --project cli -t "LocalCLI" + env: + TRA_LABEL: ${{ matrix.os }} + run: pnpm test -- --project cli -t "LocalCLI" --reporter=default --reporter=./scripts/ci-report/tra-reporter.mjs + + # Dogfooding: report this run's results to BrowserStack Test Reporting & Analytics via our own SDK. + # Best-effort only; never affects the job result. + - name: Report results to BrowserStack TRA + if: always() + continue-on-error: true + run: node scripts/ci-report/ci-report.mjs junit.xml ${{ matrix.os }} # ── 3. Publish docs (main branch only) ─────────────────────────────────────── publish-docs: diff --git a/package.json b/package.json index 3c1176d..d3ec585 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "browserstack-client", "private": true, "type": "module", - "version": "7.1.1", + "version": "7.2.0", "description": "BrowserStack client SDK monorepo", "packageManager": "pnpm@10.33.2", "engines": { diff --git a/packages/accessibility/package.json b/packages/accessibility/package.json index fab83d6..0f5b94a 100644 --- a/packages/accessibility/package.json +++ b/packages/accessibility/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "BrowserStack Accessibility API client", "engines": { diff --git a/packages/app-automate/package.json b/packages/app-automate/package.json index 0ae9402..51b09ee 100644 --- a/packages/app-automate/package.json +++ b/packages/app-automate/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "BrowserStack App Automate API client for mobile app automation testing", "engines": { diff --git a/packages/automate/package.json b/packages/automate/package.json index 83ee795..7d688e8 100644 --- a/packages/automate/package.json +++ b/packages/automate/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "BrowserStack Automate API client for browser automation testing", "engines": { diff --git a/packages/cli/typescript/package.json b/packages/cli/typescript/package.json index 84fd8e8..31beb63 100644 --- a/packages/cli/typescript/package.json +++ b/packages/cli/typescript/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "Command-line interface for BrowserStack Local Testing and App Automate", "engines": { diff --git a/packages/core/package.json b/packages/core/package.json index a1253b8..80b39d2 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "Shared runtime utilities: APIClient base, auth, error handling, HTTP adapters", "engines": { diff --git a/packages/local-testing-binary/package.json b/packages/local-testing-binary/package.json index c438cf2..76df17c 100644 --- a/packages/local-testing-binary/package.json +++ b/packages/local-testing-binary/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "BrowserStack Local Testing binary management and spawning for Node.js", "engines": { diff --git a/packages/local-testing/package.json b/packages/local-testing/package.json index c495a0c..7599967 100644 --- a/packages/local-testing/package.json +++ b/packages/local-testing/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "BrowserStack Local Testing API client for local development", "engines": { diff --git a/packages/openapi-transforms/package.json b/packages/openapi-transforms/package.json index ef8ba1a..49917e9 100644 --- a/packages/openapi-transforms/package.json +++ b/packages/openapi-transforms/package.json @@ -4,7 +4,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "main": "./dist/index.js", "module": "./dist/index.js", diff --git a/packages/openapi/package.json b/packages/openapi/package.json index c918989..fa445ab 100644 --- a/packages/openapi/package.json +++ b/packages/openapi/package.json @@ -1,7 +1,7 @@ { "name": "@dot-slash/browserstack-openapi", "private": true, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "OpenAPI specs and generated TypeScript types for BrowserStack APIs", "files": [ diff --git a/packages/screenshots/package.json b/packages/screenshots/package.json index b8c3c79..d9beb0a 100644 --- a/packages/screenshots/package.json +++ b/packages/screenshots/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "BrowserStack Screenshots API client for cross-browser screenshot automation", "engines": { diff --git a/packages/sdk/package.json b/packages/sdk/package.json index d393403..745a6b7 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "BrowserStack SDK — all API clients in one package", "engines": { diff --git a/packages/test-management/package.json b/packages/test-management/package.json index 4245384..0b247e7 100644 --- a/packages/test-management/package.json +++ b/packages/test-management/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.2", + "version": "7.2.0", "type": "module", "description": "BrowserStack Test Management API client", "engines": { diff --git a/packages/test-reporting/package.json b/packages/test-reporting/package.json index e543b77..80ed1c5 100644 --- a/packages/test-reporting/package.json +++ b/packages/test-reporting/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "BrowserStack Test Reporting & Analytics API client", "engines": { diff --git a/packages/test-reporting/src/__tests__/ingestion.live.test.ts b/packages/test-reporting/src/__tests__/ingestion.live.test.ts new file mode 100644 index 0000000..f2786ca --- /dev/null +++ b/packages/test-reporting/src/__tests__/ingestion.live.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { TestReportingClient } from "../index.ts"; + +const username = process.env.BROWSERSTACK_USERNAME; +const accessKey = process.env.BROWSERSTACK_ACCESS_KEY ?? process.env.BROWSERSTACK_KEY; + +// End to end against the real Test Reporting & Analytics ingestion API: report a run, then read it back. +describe.skipIf(!username || !accessKey)("TestReportingClient live ingestion", () => { + it("reports a build with a passed and a failed test and reads them back", async () => { + const client = new TestReportingClient({ username, accessKey }); + const now = () => new Date().toISOString(); + const { buildHashedId } = await client.startBuild({ + projectName: "sdk-integration-tests", + name: `ingestion-live ${now()}`, + startedAt: now(), + framework: { name: "vitest", version: "4" }, + tags: ["sdk-test"], + }); + expect(buildHashedId).toMatch(/^[a-z0-9]{40}$/); + + const ok = await client.startTestRun(buildHashedId!, { name: "passes", fileName: "ingestion.live.test.ts", scopes: ["live"], startedAt: now() }); + const bad = await client.startTestRun(buildHashedId!, { name: "fails", fileName: "ingestion.live.test.ts", scopes: ["live"], startedAt: now() }); + await client.addBuildLogs(buildHashedId!, { logs: [{ kind: "TEST_LOG", testRunUuid: bad.uuid, level: "ERROR", message: "boom", timestamp: now() }] }); + await client.finishTestRun(buildHashedId!, ok.uuid!, { result: "passed", finishedAt: now(), fileName: "ingestion.live.test.ts", scopes: ["live"], durationInMs: 5 }); + await client.finishTestRun(buildHashedId!, bad.uuid!, { + result: "failed", + finishedAt: now(), + fileName: "ingestion.live.test.ts", + scopes: ["live"], + durationInMs: 5, + failure: [{ error: "AssertionError: boom", backtrace: "at x" }], + }); + await client.finishBuild(buildHashedId!, { finishedAt: now() }); + + const reader = client; + let summary: { passed?: number; failed?: number } | undefined; + for (let i = 0; i < 20; i++) { + try { + summary = (await reader.getTestRuns(buildHashedId!)).testSummary; // the build can take a moment to become readable + } catch { + summary = undefined; + } + if ((summary?.passed ?? 0) + (summary?.failed ?? 0) >= 2) break; + await new Promise((r) => setTimeout(r, 3000)); + } + expect(summary).toMatchObject({ passed: 1, failed: 1 }); + }, 90_000); +}); diff --git a/packages/test-reporting/src/__tests__/ingestion.test.ts b/packages/test-reporting/src/__tests__/ingestion.test.ts new file mode 100644 index 0000000..c8d1a41 --- /dev/null +++ b/packages/test-reporting/src/__tests__/ingestion.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; +import { BrowserStackError } from "@dot-slash/browserstack-core"; +import { HttpError } from "@dot-slash/browserstack-openapi-transforms"; +import { TestReportingClient } from "../index.ts"; + +const COLLECTOR = "https://collector-observability.browserstack.com"; + +function setup(startBody: unknown = { build_hashed_id: "bld", jwt: "tok" }) { + const calls: Array<{ url: string; method: string; headers: Record; body: any }> = []; + const queue: Array<{ status?: number; body: unknown }> = [{ body: startBody }]; + const fetchFn = (async (url: string | URL | Request, init?: RequestInit) => { + calls.push({ url: url.toString(), method: init?.method ?? "GET", headers: Object.fromEntries(new Headers(init?.headers).entries()), body: init?.body ? JSON.parse(init.body as string) : undefined }); + const r = queue.shift() ?? { body: { success: true } }; + return new Response(JSON.stringify(r.body), { status: r.status ?? 200, headers: { "content-type": "application/json" } }); + }) as typeof fetch; + const client = new TestReportingClient({ username: "u", accessKey: "k", fetchFn }); + return { client, calls, queue }; +} + +const build = { name: "build", projectName: "proj", startedAt: "2026-01-01T00:00:00Z", framework: { name: "vitest", version: "4" } }; +const started = async (s = setup()) => ({ ...s, id: (await s.client.startBuild(build)).buildHashedId as string }); + +describe("live ingestion (TestReportingClient)", () => { + it("startBuild posts to the collector with basic auth and returns the build id", async () => { + const { client, calls } = setup(); + const res = await client.startBuild({ ...build, tags: ["ci"], ciInfo: { name: "GitHub Actions", buildUrl: "https://ci/1" }, versionControl: { sha: "abc", commitMessage: "m" } } as never); + expect(res).toEqual({ success: true, buildHashedId: "bld" }); + expect(calls[0].url).toBe(`${COLLECTOR}/api/v2/builds`); + expect(calls[0].headers.authorization).toBe(`Basic ${btoa("u:k")}`); + expect(calls[0].body).toMatchObject({ + format: "json", + project_name: "proj", + name: "build", + started_at: "2026-01-01T00:00:00Z", + tags: ["ci"], + ci_info: { name: "GitHub Actions", build_url: "https://ci/1" }, + version_control: { sha: "abc", commit_message: "m" }, + product_map: { observability: true }, + framework_details: { frameworkName: "vitest", frameworkVersion: "4", testFramework: { name: "vitest", version: "4" } }, + }); + }); + + it("startBuild fails without a JWT", async () => { + await expect(setup({ build_hashed_id: "bld" }).client.startBuild(build)).rejects.toThrow(/JWT/); + }); + + it("honours ingestBaseUrl", async () => { + const urls: string[] = []; + const fetchFn = (async (url: string | URL | Request) => (urls.push(url.toString()), new Response(JSON.stringify({ build_hashed_id: "b", jwt: "t" }), { status: 200 }))) as typeof fetch; + await new TestReportingClient({ username: "u", accessKey: "k", fetchFn, ingestBaseUrl: "https://example.test/" }).startBuild(build); + expect(urls[0]).toBe("https://example.test/api/v2/builds"); + }); + + it("startTestRun sends TestRunStarted with the build JWT and returns the run uuid", async () => { + const { client, calls, id } = await started(); + const res = await client.startTestRun(id, { name: "t", fileName: "a.test.ts", scopes: ["unit", "suite"], startedAt: "2026-01-01T00:00:01Z" }); + expect(res.uuid).toMatch(/^[0-9a-f-]{36}$/); + expect(calls[1].url).toBe(`${COLLECTOR}/api/v1/batch`); + expect(calls[1].headers.authorization).toBe("Bearer tok"); + expect(calls[1].body).toEqual([ + { event_type: "TestRunStarted", test_run: expect.objectContaining({ uuid: res.uuid, type: "test", name: "t", file_name: "a.test.ts", scopes: ["unit", "suite"], scope: "unit > suite", framework: "vitest", result: "pending", started_at: "2026-01-01T00:00:01Z" }) }, + ]); + }); + + it("finishTestRun sends TestRunFinished merged with the start data, including failure details", async () => { + const { client, calls, id } = await started(); + const { uuid } = await client.startTestRun(id, { name: "t", fileName: "a.test.ts", scopes: ["unit"], startedAt: "2026-01-01T00:00:01Z" }); + await client.finishTestRun(id, uuid as string, { + result: "failed", + finishedAt: "2026-01-01T00:00:02Z", + fileName: "a.test.ts", + scopes: ["unit"], + durationInMs: 1000, + failure: [{ error: "AssertionError: boom", backtrace: "at x" }], + }); + expect(calls[2].body[0]).toMatchObject({ + event_type: "TestRunFinished", + test_run: { uuid, name: "t", started_at: "2026-01-01T00:00:01Z", finished_at: "2026-01-01T00:00:02Z", result: "failed", duration_in_ms: 1000, failure: [{ backtrace: ["AssertionError: boom", "at x"] }], failure_reason: "AssertionError: boom", failure_type: "AssertionError" }, + }); + }); + + it("hook runs use HookRunStarted/HookRunFinished", async () => { + const { client, calls, id } = await started(); + const { uuid } = await client.startHookRun(id, { hookType: "BEFORE_ALL", name: "setup", fileName: "a.test.ts", scopes: ["unit"], startedAt: "2026-01-01T00:00:01Z" }); + await client.finishHookRun(id, uuid as string, { hookType: "BEFORE_ALL", result: "passed", finishedAt: "2026-01-01T00:00:02Z", fileName: "a.test.ts", scopes: ["unit"] }); + expect(calls[1].body[0]).toMatchObject({ event_type: "HookRunStarted", hook_run: { uuid, type: "hook", hook_type: "BEFORE_ALL" } }); + expect(calls[2].body[0]).toMatchObject({ event_type: "HookRunFinished", hook_run: { uuid, result: "passed" } }); + }); + + it("addBuildLogs sends a LogCreated event with snake_case fields", async () => { + const { client, calls, id } = await started(); + await client.addBuildLogs(id, { logs: [{ kind: "TEST_LOG", testRunUuid: "u1", level: "ERROR", message: "boom", timestamp: "2026-01-01T00:00:01Z" }] }); + expect(calls[1].body).toEqual([ + { event_type: "LogCreated", logs: [expect.objectContaining({ kind: "TEST_LOG", test_run_uuid: "u1", level: "ERROR", message: "boom", timestamp: "2026-01-01T00:00:01Z", http_response: {} })] }, + ]); + }); + + it("finishBuild stops the build and forgets it", async () => { + const { client, calls, id } = await started(); + await client.finishBuild(id, { finishedAt: "2026-01-01T00:01:00Z" }); + expect(calls[1].url).toBe(`${COLLECTOR}/api/v1/builds/bld/stop`); + expect(calls[1].method).toBe("PUT"); + expect(calls[1].body).toEqual({ stop_time: "2026-01-01T00:01:00Z" }); + await expect(client.addBuildLogs(id, { logs: [] })).rejects.toThrow(BrowserStackError); + }); + + it("linkTestRunSession sends CBTSessionCreated with the session details", async () => { + const { client, calls, id } = await started(); + await client.linkTestRunSession(id, "run1", { sessionId: "s1", browser: "chrome", browserVersion: "130", platform: "linux" }); + expect(calls[1].body).toEqual([ + { event_type: "CBTSessionCreated", test_run: { uuid: "run1", integrations: { browserstack: expect.objectContaining({ session_id: "s1", browser: "chrome", browser_version: "130", platform: "linux" }) } } }, + ]); + }); + + it("addTestScreenshots sends TEST_SCREENSHOT logs to the screenshots endpoint", async () => { + const { client, calls, id } = await started(); + await client.addTestScreenshots(id, [{ testRunUuid: "run1", base64: "aGk=" }]); + expect(calls[1].url).toBe(`${COLLECTOR}/api/v1/screenshots`); + expect(calls[1].body[0]).toMatchObject({ event_type: "LogCreated", logs: [expect.objectContaining({ kind: "TEST_SCREENSHOT", test_run_uuid: "run1", message: "aGk=" })] }); + }); + + it("rejects runs and logs for a build this client did not start", async () => { + const { client } = setup(); + await expect(client.startTestRun("nope", { name: "t", fileName: "f", scopes: ["s"], startedAt: "2026-01-01T00:00:00Z" })).rejects.toThrow(/startBuild/); + }); + + it("surfaces HTTP errors with status and body", async () => { + const { client, queue } = setup(); + queue[0] = { status: 403, body: { message: "no" } }; + await expect(client.startBuild(build)).rejects.toBeInstanceOf(HttpError); + }); +}); diff --git a/packages/test-reporting/src/__tests__/test-reporting.test.ts b/packages/test-reporting/src/__tests__/test-reporting.test.ts index b914342..1b93c1a 100644 --- a/packages/test-reporting/src/__tests__/test-reporting.test.ts +++ b/packages/test-reporting/src/__tests__/test-reporting.test.ts @@ -31,11 +31,8 @@ const BUILD_DETAIL_WIRE = { project_id: 25033, }; -const START_BUILD_WIRE = { success: true, build_hashed_id: "bld_newbuild123" }; -const FINISH_BUILD_WIRE = { success: true, message: "Build finished successfully." }; -const START_TEST_RUN_WIRE = { success: true, test_run_id: "tr-001" }; describe("TestReportingClient", () => { describe("Credentials", () => { @@ -168,62 +165,6 @@ describe("TestReportingClient", () => { }); }); - describe("startBuild", () => { - it("returns success true and buildHashedId", async () => { - const client = makeClient(START_BUILD_WIRE); - const data = await client.startBuild({ - name: "my-build", - projectName: "sdk-integration-tests", - startedAt: "2026-04-27T10:00:00.000Z", - framework: { name: "vitest", version: "1.0.0" }, - }); - expect(data).toBeDefined(); - expect((data as { success?: boolean }).success).toBe(true); - expect((data as { buildHashedId?: string }).buildHashedId).toBe("bld_newbuild123"); - }); - - it("throws HttpError on 422", async () => { - const client = makeClient(makeErrorResponse(422, "Unprocessable Entity")); - await expect( - client.startBuild({ name: "", projectName: "", startedAt: "", framework: { name: "vitest", version: "1.0.0" } }) - ).rejects.toThrow(HttpError); - }); - }); - - describe("finishBuild", () => { - it("returns success true and message", async () => { - const client = makeClient(FINISH_BUILD_WIRE); - const data = await client.finishBuild("bld_newbuild123", { - finishedAt: "2026-04-27T10:30:00.000Z", - }); - expect(data).toBeDefined(); - expect((data as { success?: boolean }).success).toBe(true); - expect((data as { message?: string }).message).toBe("Build finished successfully."); - }); - - it("throws HttpError on 401", async () => { - const client = makeClient(makeErrorResponse(401, "Unauthorized")); - await expect( - client.finishBuild("bld_newbuild123", { finishedAt: "2026-04-27T10:30:00.000Z" }) - ).rejects.toThrow(HttpError); - }); - }); - - describe("startTestRun", () => { - it("returns success true and test_run_id", async () => { - const client = makeClient(START_TEST_RUN_WIRE); - const data = await client.startTestRun("bld_newbuild123", { - name: "sdk test run", - startedAt: "2026-04-27T10:00:00.000Z", - fileName: "sdk.test.ts", - scopes: ["integration"], - }); - expect(data).toBeDefined(); - expect((data as { success?: boolean }).success).toBe(true); - expect((data as { testRunId?: string }).testRunId).toBe("tr-001"); - }); - }); - describe("uploadReport", () => { function makeCapturingClient() { let capturedUrl = ""; diff --git a/packages/test-reporting/src/index.ts b/packages/test-reporting/src/index.ts index da16c16..7dfdf49 100644 --- a/packages/test-reporting/src/index.ts +++ b/packages/test-reporting/src/index.ts @@ -1,7 +1,18 @@ -import { BrowserStackOptions } from "@dot-slash/browserstack-core"; +import { BrowserStackError, BrowserStackOptions } from "@dot-slash/browserstack-core"; import { AppAutomateClient } from "@dot-slash/browserstack-app-automate"; import { AutomateClient } from "@dot-slash/browserstack-automate"; +import { HttpError, NetworkError } from "@dot-slash/browserstack-openapi-transforms"; import { GeneratedTestReportingClient } from "@dot-slash/browserstack-openapi/test-reporting/client"; +import { + buildStartBody, + failureFields, + logFields, + runFields, + sessionIntegrations, + type LogEntry, + type RunSession, + type RunStart, +} from "./ingestion"; import { defaultKinds, fetchAppAutomateLogs, @@ -18,9 +29,12 @@ import { } from "./linked-sessions"; export type * from "./linked-sessions"; +export type { LogEntry, RunSession } from "./ingestion"; export interface TestReportingClientOptions extends BrowserStackOptions { uploadBaseUrl?: string; + /** Base URL of the live ingestion API (build start/stop, test and hook events, logs). */ + ingestBaseUrl?: string; /** * Used to resolve a test's session and fetch its logs (see `getTestSession`). Defaults to a client built from * these same options, minus the Test Reporting base URLs. Pass one to customise it (base URL, timeout, ...). @@ -34,6 +48,9 @@ export class TestReportingClient extends GeneratedTestReportingClient { private readonly siblingOptions: BrowserStackOptions; private automateClient: AutomateClient | undefined; private appAutomateClient: AppAutomateClient | undefined; + private readonly ingestBaseUrl: string; + /** Per started build: its ingestion JWT, framework, and the runs that have started but not finished. */ + private readonly builds = new Map }>(); private readonly sessionCache = new Map>(); constructor(options?: TestReportingClientOptions) { @@ -44,12 +61,14 @@ export class TestReportingClient extends GeneratedTestReportingClient { "@dot-slash/browserstack-test-reporting", __PKG_VERSION__ ); + this.ingestBaseUrl = (options?.ingestBaseUrl ?? "https://collector-observability.browserstack.com").replace(/\/$/, ""); const shared: TestReportingClientOptions = { ...options }; this.automateClient = shared.automate; this.appAutomateClient = shared.appAutomate; // Those base URLs are Test Reporting's, so the sibling clients keep their own. delete shared.baseUrl; delete shared.uploadBaseUrl; + delete shared.ingestBaseUrl; delete shared.automate; delete shared.appAutomate; this.siblingOptions = shared; @@ -126,6 +145,137 @@ export class TestReportingClient extends GeneratedTestReportingClient { : fetchAppAutomateLogs(this.appAutomate, linked.session, sessionId, wanted); } + // ── Live ingestion ──────────────────────────────────────────────────────────────────────────────────────────── + // These override the generated ingestion methods (same signatures) so they work against the live service, which + // is the protocol BrowserStack's SDKs use: start a build with basic auth (returns a build id and a JWT), stream + // events with that JWT, stop the build. A client must start a build before reporting runs or logs into it. + + private async ingest(operationId: string, method: "POST" | "PUT", path: string, authorization: string, body: unknown): Promise { + const url = `${this.ingestBaseUrl}${path}`; + const ctx = { operationId, method, url }; + let res: Response; + try { + res = await this.fetchFn(url, { + method, + headers: { "content-type": "application/json", authorization, "user-agent": this.userAgent }, + body: JSON.stringify(body), + }); + } catch (cause) { + throw new NetworkError((cause as Error).message || "network error", ctx, cause as Error); + } + const text = await res.text(); + if (!res.ok) { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + parsed = undefined; + } + const message = (parsed as { message?: string } | undefined)?.message ?? `HTTP ${res.status} ${res.statusText}`; + throw new HttpError(message, ctx, { + status: res.status, + statusText: res.statusText, + headers: res.headers, + body: { text, parsed, truncated: false } as never, + retryable: res.status >= 500 || res.status === 429, + }); + } + return (text ? JSON.parse(text) : {}) as T; + } + + private buildState(buildHashedId: string) { + const b = this.builds.get(buildHashedId); + if (!b) throw new BrowserStackError(`Build ${buildHashedId} was not started by this client; call startBuild() first`); + return b; + } + + private async sendEvents(buildHashedId: string, events: unknown[], path = "/api/v1/batch"): Promise { + await this.ingest("sendTestReportingEvents", "POST", path, `Bearer ${this.buildState(buildHashedId).jwt}`, events); + } + + override async startBuild(body: Parameters[0]): ReturnType { + const res = await this.ingest<{ build_hashed_id?: string; jwt?: string }>("startTestReportingBuild", "POST", "/api/v2/builds", this.authHeader ?? "", buildStartBody(body as Parameters[0])); + if (!res.build_hashed_id || !res.jwt) throw new BrowserStackError("Build start returned no build id or JWT (is Test Reporting enabled for this account?)"); + this.builds.set(res.build_hashed_id, { jwt: res.jwt, framework: (body as { framework?: { name: string } }).framework?.name, runs: new Map() }); + return { success: true, buildHashedId: res.build_hashed_id } as Awaited>; + } + + override async startTestRun(buildHashedId: string, body: Parameters[1]): ReturnType { + return this.startRun("test", buildHashedId, body) as ReturnType; + } + + override async finishTestRun(buildHashedId: string, testRunUuid: string, body: Parameters[2]): ReturnType { + return this.finishRun("test", buildHashedId, testRunUuid, body) as ReturnType; + } + + override async startHookRun(buildHashedId: string, body: Parameters[1]): ReturnType { + return this.startRun("hook", buildHashedId, body) as ReturnType; + } + + override async finishHookRun(buildHashedId: string, hookRunUuid: string, body: Parameters[2]): ReturnType { + return this.finishRun("hook", buildHashedId, hookRunUuid, body) as ReturnType; + } + + private async startRun(type: "test" | "hook", buildHashedId: string, body: Record) { + const build = this.buildState(buildHashedId); + const run: RunStart = { + type, + uuid: crypto.randomUUID(), + name: body.name, + fileName: body.fileName, + scopes: body.scopes, + startedAt: body.startedAt, + tags: body.tags, + location: body.location, + hookType: body.hookType, + framework: build.framework, + }; + const data = runFields(run, { result: body.result ?? "pending", environment: body.environment, custom_metadata: body.customMetadata, test_run_id: body.testRunId }); + await this.sendEvents(buildHashedId, [{ event_type: type === "test" ? "TestRunStarted" : "HookRunStarted", [type === "test" ? "test_run" : "hook_run"]: data }]); + build.runs.set(run.uuid, run); + return { success: true, uuid: run.uuid }; + } + + private async finishRun(type: "test" | "hook", buildHashedId: string, uuid: string, body: Record) { + const build = this.buildState(buildHashedId); + const run = build.runs.get(uuid); + if (!run) throw new BrowserStackError(`${type} run ${uuid} was not started by this client`); + const data = runFields( + { ...run, fileName: body.fileName ?? run.fileName, scopes: body.scopes ?? run.scopes }, + { finished_at: body.finishedAt, result: body.result, duration_in_ms: body.durationInMs, environment: body.environment, custom_metadata: body.customMetadata, ...failureFields(body.failure) } + ); + await this.sendEvents(buildHashedId, [{ event_type: type === "test" ? "TestRunFinished" : "HookRunFinished", [type === "test" ? "test_run" : "hook_run"]: data }]); + build.runs.delete(uuid); + return { success: true, message: `${type === "test" ? "Test" : "Hook"} run updated successfully.` }; + } + + override async addBuildLogs(buildHashedId: string, body: Parameters[1]): ReturnType { + const logs = (body.logs as unknown as LogEntry[]).map(logFields); + await this.sendEvents(buildHashedId, [{ event_type: "LogCreated", logs }]); + return { success: true, message: "Logs ingested successfully." } as Awaited>; + } + + override async finishBuild(buildHashedId: string, body: Parameters[1]): ReturnType { + const build = this.buildState(buildHashedId); + await this.ingest("finishTestReportingBuild", "PUT", `/api/v1/builds/${encodeURIComponent(buildHashedId)}/stop`, `Bearer ${build.jwt}`, { stop_time: body.finishedAt }); + this.builds.delete(buildHashedId); + return { success: true, message: "Build finished successfully." } as Awaited>; + } + + /** + * Links a test run to the BrowserStack session it ran in, so TRA can show its session, logs and video (what + * WebdriverIO sends as a CBTSessionCreated event). Takes the uuid `startTestRun` returned. + */ + async linkTestRunSession(buildHashedId: string, testRunUuid: string, session: RunSession): Promise { + await this.sendEvents(buildHashedId, [{ event_type: "CBTSessionCreated", test_run: { uuid: testRunUuid, integrations: sessionIntegrations(session) } }]); + } + + /** Attaches screenshots to test runs: `kind` `TEST_SCREENSHOT` logs whose `message` is the base64 image. */ + async addTestScreenshots(buildHashedId: string, screenshots: Array<{ testRunUuid: string; base64: string; timestamp?: string }>): Promise { + const logs = screenshots.map((s) => logFields({ kind: "TEST_SCREENSHOT", testRunUuid: s.testRunUuid, message: s.base64, timestamp: s.timestamp ?? new Date().toISOString() })); + await this.sendEvents(buildHashedId, [{ event_type: "LogCreated", logs }], "/api/v1/screenshots"); + } + private async lookupSession(sessionId: string, hint?: SessionHint): Promise { const web = async (): Promise => { try { diff --git a/packages/test-reporting/src/ingestion.ts b/packages/test-reporting/src/ingestion.ts new file mode 100644 index 0000000..e99952e --- /dev/null +++ b/packages/test-reporting/src/ingestion.ts @@ -0,0 +1,143 @@ +// Live ingestion into Test Reporting & Analytics. This is the protocol the BrowserStack SDKs (e.g. WebdriverIO) use: +// a build is started with basic auth (which returns a build id and a JWT), test/hook/log events are streamed to the +// batch endpoint with that JWT, and the build is stopped at the end. The mapping from the camelCase request bodies of +// `TestReportingClient` to those wire events lives here. + +const snake = (k: string) => k.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`); +const snakeKeys = (o: Record | undefined) => o && Object.fromEntries(Object.entries(o).map(([k, v]) => [snake(k), v])); + +export interface FrameworkInfo { + name: string; + version: string; +} + +export function buildStartBody(body: { + name: string; + projectName: string; + startedAt: string; + framework?: FrameworkInfo; + tags?: string[]; + buildRunIdentifier?: string; + hostInfo?: Record; + ciInfo?: Record; + versionControl?: Record; +}) { + const f = body.framework; + return { + format: "json", + project_name: body.projectName, + name: body.name, + started_at: body.startedAt, + tags: body.tags, + build_run_identifier: body.buildRunIdentifier, + host_info: snakeKeys(body.hostInfo), + ci_info: snakeKeys(body.ciInfo), + version_control: snakeKeys(body.versionControl), + framework_details: f && { + frameworkName: f.name, + frameworkVersion: f.version, + sdkVersion: f.version, + language: "ECMAScript", + testFramework: { name: f.name, version: f.version }, + }, + product_map: { observability: true }, + config: {}, + }; +} + +export interface RunStart { + type: "test" | "hook"; + uuid: string; + name: string; + fileName: string; + scopes: string[]; + startedAt: string; + tags?: string[]; + location?: string; + hookType?: string; + framework?: string; +} + +/** Run fields that stay the same between a run's start and finish event. */ +export function runFields(r: RunStart, extra: Record = {}) { + return { + uuid: r.uuid, + type: r.type, + name: r.name, + scope: r.scopes.join(" > "), + scopes: r.scopes, + identifier: `${r.fileName} > ${r.scopes.join(" > ")} > ${r.name}`, + file_name: r.fileName, + location: r.location ?? r.fileName, + tags: r.tags, + framework: r.framework, + hook_type: r.hookType, + started_at: r.startedAt, + ...extra, + }; +} + +export function failureFields(failure: Array<{ error?: string; backtrace?: string }> | undefined) { + if (!failure?.length) return {}; + const first = failure[0].error ?? ""; + return { + failure: failure.map((x) => ({ backtrace: [x.error ?? "", x.backtrace ?? ""] })), + failure_reason: first, + failure_type: /AssertionError/.test(first) ? "AssertionError" : "UnhandledError", + }; +} + +export interface LogEntry { + kind: string; + testRunUuid?: string; + hookRunUuid?: string; + timestamp?: string; + level?: string; + message?: string; + duration?: number; + failure?: boolean; + fileName?: string; + fileSize?: number; + attachmentType?: string; + httpResponse?: Record; +} + +export const logFields = (l: LogEntry) => ({ + kind: l.kind, + test_run_uuid: l.testRunUuid, + hook_run_uuid: l.hookRunUuid, + timestamp: l.timestamp, + level: l.level, + message: l.message, + duration: l.duration, + failure: l.failure, + file_name: l.fileName, + file_size: l.fileSize, + attachment_type: l.attachmentType, + http_response: l.httpResponse ?? {}, +}); + +/** Session/cloud details of a test run, as BrowserStack's CBTSessionCreated event carries them. */ +export interface RunSession { + /** Cloud provider key, e.g. `browserstack`. */ + provider?: string; + sessionId: string; + browser?: string; + browserVersion?: string; + platform?: string; + platformVersion?: string; + device?: string; + capabilities?: Record; +} + +export const sessionIntegrations = (s: RunSession) => ({ + [s.provider ?? "browserstack"]: { + capabilities: s.capabilities, + session_id: s.sessionId, + browser: s.browser, + browser_version: s.browserVersion, + platform: s.platform, + platform_version: s.platformVersion, + device: s.device, + }, +}); diff --git a/packages/website-scanner/package.json b/packages/website-scanner/package.json index 46eadf4..e319294 100644 --- a/packages/website-scanner/package.json +++ b/packages/website-scanner/package.json @@ -3,7 +3,7 @@ "publishConfig": { "access": "public" }, - "version": "7.1.1", + "version": "7.2.0", "type": "module", "description": "BrowserStack Website Scanner API client", "engines": { diff --git a/scripts/ci-report/tra-reporter.mjs b/scripts/ci-report/tra-reporter.mjs new file mode 100644 index 0000000..c0aa6a1 --- /dev/null +++ b/scripts/ci-report/tra-reporter.mjs @@ -0,0 +1,182 @@ +// Vitest reporter that streams this repo's CI test results to BrowserStack Test Reporting & Analytics, live, +// through our own TestReportingClient (dogfooding): the build starts with the run, each test is started when it +// is ready and finished when it has a result, and the build is finished at the end. +// Best-effort: every API error is logged and swallowed, so an unreachable BrowserStack never fails CI. +import { relative } from "node:path"; +import os from "node:os"; +import { readFileSync } from "node:fs"; + +const warn = (what, e) => console.warn(`[tra] ${what} failed (ignored): ${e instanceof Error ? e.message : e}`); +const iso = () => new Date().toISOString(); + +export class TraReporter { + constructor({ client, env = process.env, cwd = process.cwd(), event } = {}) { + this.client = client; + this.env = env; + this.cwd = cwd; + // The GitHub Actions event payload (push / pull_request) has the commit and PR details env vars lack. + this.event = event ?? (env.GITHUB_EVENT_PATH ? JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf-8")) : {}); + this.enabled = Boolean(client); + this.build = undefined; // Promise (buildHashedId) + this.tests = new Map(); // test id -> Promise<{ buildId, uuid } | undefined> + this.logs = []; // pending build log entries, flushed at the end of the run + } + + static async fromEnv(env = process.env) { + const username = env.BROWSERSTACK_USERNAME; + const accessKey = env.BROWSERSTACK_ACCESS_KEY || env.BROWSERSTACK_KEY; + if (!username || !accessKey) return new TraReporter({ env }); + const { TestReportingClient } = await import("../../packages/test-reporting/dist/index.js"); + return new TraReporter({ client: new TestReportingClient({ username, accessKey }), env }); + } + + buildRequest() { + const e = this.env; + const ev = this.event; + const pr = ev.pull_request; + const commit = ev.head_commit; + const repo = e.GITHUB_REPOSITORY; + const runId = e.GITHUB_RUN_ID; + const label = e.TRA_LABEL; + // On pull_request runs GITHUB_SHA / GITHUB_REF_NAME describe the synthetic merge commit; report the PR head. + const branch = pr?.head?.ref ?? e.GITHUB_HEAD_REF ?? e.GITHUB_REF_NAME; + const sha = pr?.head?.sha ?? e.GITHUB_SHA; + const server = e.GITHUB_SERVER_URL ?? "https://github.com"; + const req = { + name: runId ? `${e.GITHUB_WORKFLOW ?? "CI"} #${e.GITHUB_RUN_NUMBER} (${branch})${label ? ` [${label}]` : ""}` : `local ${iso()}`, + projectName: repo ?? "browserstack-client", + startedAt: iso(), + tags: [...new Set(["ci", label, branch, e.GITHUB_EVENT_NAME].filter(Boolean))], + framework: { name: "vitest", version: this.vitestVersion ?? "unknown" }, + hostInfo: { hostname: os.hostname(), platform: os.platform(), arch: os.arch(), version: os.release(), type: e.RUNNER_OS }, + }; + if (runId) { + req.buildRunIdentifier = [runId, e.GITHUB_RUN_ATTEMPT ?? "1", label].filter(Boolean).join("-"); + req.ciInfo = { + name: "GitHub Actions", + buildUrl: `${server}/${repo}/actions/runs/${runId}`, + url: pr?.html_url ?? `${server}/${repo}/commit/${sha}`, + buildNumber: e.GITHUB_RUN_NUMBER, + jobName: e.GITHUB_JOB, + }; + } + if (sha) { + req.versionControl = { + name: "git", + sha, + branch, + commitMessage: (pr?.title ?? commit?.message)?.split("\n")[0], + committerName: commit?.committer?.name ?? e.GITHUB_ACTOR, + committerEmail: commit?.committer?.email, + }; + } + return req; + } + + async onTestRunStart() { + if (!this.enabled) return; + this.build = this.client + .startBuild(this.buildRequest()) + .then((res) => res?.buildHashedId ?? res?.build_hashed_id) + .catch((e) => (warn("startBuild", e), undefined)); + await this.build; + } + + async onTestCaseReady(testCase) { + if (!this.enabled) return; + const started = (async () => { + const buildId = await this.build; + if (!buildId) return undefined; + try { + const parts = testCase.fullName.split(" > "); + const res = await this.client.startTestRun(buildId, { + name: testCase.name, + fileName: relative(this.cwd, testCase.module.moduleId).replaceAll("\\", "/"), + scopes: [testCase.project.name, ...parts.slice(0, -1)].slice(0, 20), + startedAt: iso(), + ...(testCase.location ? { location: `${testCase.location.line}:${testCase.location.column}` } : {}), + }); + const uuid = res?.uuid ?? res?.testRunId; + return uuid ? { buildId, uuid, startedAt: Date.now() } : undefined; + } catch (e) { + warn("startTestRun", e); + return undefined; + } + })(); + this.tests.set(testCase.id, started); + await started; + } + + async onUserConsoleLog(log) { + if (!this.enabled || !log.taskId) return; + const run = await this.tests.get(log.taskId); + if (!run) return; + this.logs.push({ + kind: "TEST_LOG", + testRunUuid: run.uuid, + level: log.type === "stderr" ? "ERROR" : "INFO", + message: String(log.content).replace(/\n$/, "").slice(0, 10000), + timestamp: new Date(log.time ?? Date.now()).toISOString(), + }); + } + + async onTestCaseResult(testCase) { + if (!this.enabled) return; + const run = await this.tests.get(testCase.id); + if (!run) return; + try { + const result = testCase.result(); + const state = result.state === "pending" ? "skipped" : result.state; + const parts = testCase.fullName.split(" > "); + if (state === "failed") { + for (const x of result.errors ?? []) { + this.logs.push({ kind: "TEST_LOG", testRunUuid: run.uuid, level: "ERROR", message: String(x.stack ?? x.message).slice(0, 10000), timestamp: iso(), failure: true }); + } + } + await this.client.finishTestRun(run.buildId, run.uuid, { + result: state, + finishedAt: iso(), + fileName: relative(this.cwd, testCase.module.moduleId).replaceAll("\\", "/"), + scopes: [testCase.project.name, ...parts.slice(0, -1)].slice(0, 20), + durationInMs: Math.round(testCase.diagnostic()?.duration ?? Date.now() - run.startedAt), + ...(state === "failed" ? { failure: (result.errors ?? []).slice(0, 100).map((x) => ({ error: x.message, backtrace: x.stack })) } : {}), + }); + } catch (e) { + warn("finishTestRun", e); + } + } + + async onTestRunEnd(_modules, errors = []) { + if (!this.enabled) return; + await Promise.all(this.tests.values()); + const buildId = await this.build; + if (!buildId) return; + // Unhandled errors (outside any test) belong to the build itself. + for (const x of errors) { + this.logs.push({ kind: "TEST_LOG", level: "ERROR", message: String(x.stack ?? x.message).slice(0, 10000), timestamp: iso(), failure: true }); + } + for (let i = 0; i < this.logs.length; i += 500) { + try { + await this.client.addBuildLogs(buildId, { logs: this.logs.slice(i, i + 500) }); + } catch (e) { + warn("addBuildLogs", e); + } + } + try { + await this.client.finishBuild(buildId, { finishedAt: iso() }); + } catch (e) { + warn("finishBuild", e); + } + } +} + +// Vitest loads the default export as the reporter class; it needs the credentialed client up front. +export default class TraVitestReporter extends TraReporter { + async onInit(vitest) { + this.vitestVersion = vitest?.version; + const r = await TraReporter.fromEnv(process.env); + this.client = r.client; + this.enabled = r.enabled; + if (!this.enabled) console.log("[tra] BrowserStack credentials not available; live reporting disabled."); + } +} diff --git a/scripts/ci-report/tra-reporter.test.ts b/scripts/ci-report/tra-reporter.test.ts new file mode 100644 index 0000000..31e94e5 --- /dev/null +++ b/scripts/ci-report/tra-reporter.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, it, vi } from "vitest"; +import { TraReporter } from "./tra-reporter.mjs"; + +const env = { + GITHUB_REPOSITORY: "shirish87/browserstack-client", + GITHUB_RUN_ID: "123", + GITHUB_RUN_NUMBER: "45", + GITHUB_RUN_ATTEMPT: "1", + GITHUB_REF_NAME: "main", + GITHUB_SHA: "abcdef1234567890", + GITHUB_WORKFLOW: "CI", + TRA_LABEL: "linux", +}; + +function fakeClient() { + return { + startBuild: vi.fn(async () => ({ success: true, buildHashedId: "bld1" })), + startTestRun: vi.fn(async () => ({ success: true, uuid: "run1" })), + finishTestRun: vi.fn(async () => ({ success: true })), + finishBuild: vi.fn(async () => ({ success: true })), + addBuildLogs: vi.fn(async () => ({ success: true })), + }; +} + +const tc = (over: Record = {}) => ({ + id: "t1", + name: "does a thing", + fullName: "suite > does a thing", + module: { moduleId: "/repo/packages/core/src/a.test.ts" }, + project: { name: "core" }, + location: { line: 3, column: 1 }, + result: () => ({ state: "passed", errors: [] }), + diagnostic: () => ({ duration: 12.4 }), + ...over, +}); + +describe("TraReporter", () => { + it("starts the build when the run starts, with CI metadata", async () => { + const client = fakeClient(); + const r = new TraReporter({ client, env, cwd: "/repo" }); + await r.onTestRunStart([]); + await r.onTestRunEnd([], []); + expect(client.startBuild).toHaveBeenCalledWith( + expect.objectContaining({ + projectName: "shirish87/browserstack-client", + name: "CI #45 (main) [linux]", + buildRunIdentifier: "123-1-linux", + framework: expect.objectContaining({ name: "vitest" }), + ciInfo: expect.objectContaining({ buildUrl: "https://github.com/shirish87/browserstack-client/actions/runs/123" }), + versionControl: expect.objectContaining({ sha: "abcdef1234567890", branch: "main" }), + }) + ); + }); + + it("captures PR context: head commit and branch, PR number/title, actor, event and run/job URLs", async () => { + const client = fakeClient(); + const event = { + pull_request: { + number: 34, + title: "ci: report to TRA", + html_url: "https://github.com/shirish87/browserstack-client/pull/34", + head: { ref: "feature/x", sha: "headsha1234567" }, + }, + }; + const prEnv = { ...env, GITHUB_EVENT_NAME: "pull_request", GITHUB_HEAD_REF: "feature/x", GITHUB_ACTOR: "shirish87", GITHUB_JOB: "test-sdk", GITHUB_REF_NAME: "34/merge", GITHUB_SERVER_URL: "https://github.com", RUNNER_OS: "Linux" }; + const r = new TraReporter({ client, env: prEnv, cwd: "/repo", event }); + await r.onTestRunStart(); + expect(client.startBuild).toHaveBeenCalledWith( + expect.objectContaining({ + name: "CI #45 (feature/x) [linux]", + tags: expect.arrayContaining(["ci", "linux", "feature/x", "pull_request"]), + versionControl: expect.objectContaining({ sha: "headsha1234567", branch: "feature/x", commitMessage: "ci: report to TRA" }), + ciInfo: expect.objectContaining({ + name: "GitHub Actions", + buildUrl: "https://github.com/shirish87/browserstack-client/actions/runs/123", + url: "https://github.com/shirish87/browserstack-client/pull/34", + buildNumber: "45", + jobName: "test-sdk", + }), + }) + ); + }); + + it("captures push context: head commit message and author from the event", async () => { + const client = fakeClient(); + const event = { head_commit: { message: "fix: a thing\n\nbody", author: { name: "Ann", email: "ann@x.io" }, committer: { name: "Bob", email: "bob@x.io" } } }; + const r = new TraReporter({ client, env: { ...env, GITHUB_EVENT_NAME: "push" }, cwd: "/repo", event }); + await r.onTestRunStart(); + expect(client.startBuild).toHaveBeenCalledWith( + expect.objectContaining({ + versionControl: expect.objectContaining({ sha: "abcdef1234567890", branch: "main", commitMessage: "fix: a thing", committerName: "Bob", committerEmail: "bob@x.io" }), + }) + ); + }); + + it("reports each test live: start when it is ready, finish when it has a result", async () => { + const client = fakeClient(); + const r = new TraReporter({ client, env, cwd: "/repo" }); + await r.onTestRunStart([]); + await r.onTestCaseReady(tc()); + expect(client.startTestRun).toHaveBeenCalledWith("bld1", expect.objectContaining({ name: "does a thing", scopes: ["core", "suite"], fileName: "packages/core/src/a.test.ts" })); + expect(client.finishTestRun).not.toHaveBeenCalled(); + await r.onTestCaseResult(tc()); + expect(client.finishTestRun).toHaveBeenCalledWith("bld1", "run1", expect.objectContaining({ result: "passed", durationInMs: 12 })); + await r.onTestRunEnd([], []); + expect(client.finishBuild).toHaveBeenCalledWith("bld1", expect.objectContaining({ finishedAt: expect.any(String) })); + }); + + it("maps failures with error and backtrace", async () => { + const client = fakeClient(); + const r = new TraReporter({ client, env, cwd: "/repo" }); + await r.onTestRunStart([]); + await r.onTestCaseReady(tc()); + await r.onTestCaseResult(tc({ result: () => ({ state: "failed", errors: [{ message: "boom", stack: "at x" }] }) })); + expect(client.finishTestRun).toHaveBeenCalledWith("bld1", "run1", expect.objectContaining({ result: "failed", failure: [{ error: "boom", backtrace: "at x" }] })); + }); + + it("sends test console output as TEST_LOG entries linked to the test run", async () => { + const client = fakeClient(); + const r = new TraReporter({ client, env, cwd: "/repo" }); + await r.onTestRunStart(); + await r.onTestCaseReady(tc()); + await r.onUserConsoleLog({ content: "hello\n", type: "stderr", taskId: "t1", time: 1759600000000 }); + await r.onTestRunEnd([], []); + expect(client.addBuildLogs).toHaveBeenCalledWith("bld1", { + logs: [expect.objectContaining({ kind: "TEST_LOG", testRunUuid: "run1", level: "ERROR", message: "hello", timestamp: new Date(1759600000000).toISOString() })], + }); + }); + + it("sends failure messages and unhandled errors as ERROR logs with stack traces", async () => { + const client = fakeClient(); + const r = new TraReporter({ client, env, cwd: "/repo" }); + await r.onTestRunStart(); + await r.onTestCaseReady(tc()); + await r.onTestCaseResult(tc({ result: () => ({ state: "failed", errors: [{ message: "boom", stack: "Error: boom\n at x" }] }) })); + await r.onTestRunEnd([], [{ name: "Error", message: "unhandled", stack: "Error: unhandled\n at y" }]); + const logs = client.addBuildLogs.mock.calls.flatMap((c) => (c[1] as { logs: unknown[] }).logs); + expect(logs).toContainEqual(expect.objectContaining({ kind: "TEST_LOG", testRunUuid: "run1", level: "ERROR", message: "Error: boom\n at x", failure: true })); + expect(logs).toContainEqual(expect.objectContaining({ kind: "TEST_LOG", level: "ERROR", message: "Error: unhandled\n at y", failure: true })); + }); + + it("never throws and stops calling once the build cannot be started", async () => { + const client = fakeClient(); + client.startBuild.mockRejectedValue(new Error("unreachable")); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const r = new TraReporter({ client, env, cwd: "/repo" }); + await expect(r.onTestRunStart([])).resolves.toBeUndefined(); + await expect(r.onTestCaseReady(tc())).resolves.toBeUndefined(); + await expect(r.onTestCaseResult(tc())).resolves.toBeUndefined(); + await expect(r.onTestRunEnd([], [])).resolves.toBeUndefined(); + expect(client.startTestRun).not.toHaveBeenCalled(); + expect(client.finishBuild).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it("swallows per-test API errors", async () => { + const client = fakeClient(); + client.startTestRun.mockRejectedValue(new Error("500")); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const r = new TraReporter({ client, env, cwd: "/repo" }); + await r.onTestRunStart([]); + await r.onTestCaseReady(tc()); + await expect(r.onTestCaseResult(tc())).resolves.toBeUndefined(); + expect(client.finishTestRun).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it("is disabled without credentials", async () => { + const r = await TraReporter.fromEnv({ GITHUB_RUN_ID: "1" }); + expect(r.enabled).toBe(false); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 7063929..11a6d7f 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -85,6 +85,8 @@ export default defineConfig({ project("test-reporting", "./packages/test-reporting", "packages/test-reporting/package.json"), project("sdk", "./packages/sdk", "packages/sdk/package.json"), project("router", "./packages/router", "packages/router/package.json"), + // CI reporting script (uploads vitest JUnit results to Test Reporting & Analytics). + { test: { name: "ci-report", root: "./scripts/ci-report", globals: true, environment: "node" } }, // Offline checks of the contract tooling (sanitiser, fixtures vs generated models). { ...project("contract-tests", "./packages/contract-tests", "packages/contract-tests/package.json"),