Say why /research web search setup fails, and log the cause

README.md

@@ -185,6 +185,11 @@ Compose project.
 Set `AGENT=off` to prevent Planner turns and disable the background-job runner.
 This does not disable the `/mcp` endpoint used by external coding agents.
 
+`/research` public web search calls GitHub's Copilot MCP endpoint with the signed-in
+user's App user access token. Whether that endpoint accepts a self-registered App's
+tokens for `web_search` is unconfirmed. If it refuses, the research card and server log
+say so, and [Self-hosting](docs/self-hosting.md#troubleshooting) lists what to check.
+
 ## Collaborate with the Planner
 
 The web composer treats `@chopin` as an instruction for the Planner:

apps/server/src/harness/web-search.test.ts

@@ -86,23 +86,195 @@ it("loads only the host GitHub MCP web_search and binds execution to the owner",
 it("fails closed when MCP does not offer web_search or connection fails", async () => {
 	let closed = 0;
 	let missing = await webSearchTool(credential(), {
+		log: () => {},
 		createClient: async () => ({
 			tools: async () => ({}),
 			close: async () => {
 				closed++;
 			},
 		}),
 	});
-	expect(missing).toEqual({ ok: false, error: { kind: "MissingTool", name: "web_search" } });
+	expect(missing).toMatchObject({ ok: false, error: { kind: "MissingTool", name: "web_search" } });
 	expect(closed).toBe(1);
 	let unavailable = await webSearchTool(credential(), {
+		log: () => {},
 		createClient: async () => {
 			throw new Error("offline");
 		},
 	});
 	expect(unavailable).toMatchObject({ ok: false, error: { kind: "Unavailable" } });
 });
 
+it("names a credential the endpoint refuses at connect, with its type and a redacted message", async () => {
+	let secret = "ghu_abcdefghijklmnopqrstuvwxyz0123456789";
+	let logs: string[] = [];
+	let result = await webSearchTool({ ...credential(), token: secret }, {
+		log: line => logs.push(line),
+		createClient: async () => {
+			throw new MCPClientError({
+				message: "MCP HTTP Transport Error: POSTing to endpoint (HTTP 401): "
+					+ `unauthorized: token authentication failed for Bearer ${secret}`,
+				statusCode: 401,
+				url: "https://api.githubcopilot.com/mcp/",
+			});
+		},
+	});
+	expect(result).toMatchObject({
+		ok: false,
+		error: {
+			kind: "Unavailable",
+			failure: {
+				reason: "web-search-credential-rejected",
+				classification: "local:connect:mcp-client:http:401",
+				durationMs: expect.any(Number),
+			},
+		},
+	});
+	expect(logs).toHaveLength(1);
+	expect(logs[0]).toStartWith("chopin: web_search unavailable at connect - ");
+	expect(logs[0]).toContain("local:connect:mcp-client:http:401");
+	expect(logs[0]).toContain("GitHub App user access token (ghu_)");
+	expect(logs[0]).toContain("unauthorized: token authentication failed");
+	expect(logs[0]).not.toContain(secret);
+	expect(logs[0]).not.toContain("abcdefghijkl");
+	expect(logs[0]).not.toContain("\n");
+});
+
+it("reads a refusal while listing tools as the same rejected credential", async () => {
+	let logs: string[] = [];
+	let closed = 0;
+	let result = await webSearchTool({ ...credential(), token: "gho_0123456789abcdefghijklmnop" }, {
+		log: line => logs.push(line),
+		createClient: async () => ({
+			tools: async () => {
+				throw new MCPClientError({
+					message: "MCP HTTP Transport Error: POSTing to endpoint (HTTP 403): forbidden",
+					statusCode: 403,
+					code: -32001,
+				});
+			},
+			close: async () => {
+				closed++;
+			},
+		}),
+	});
+	expect(result).toMatchObject({
+		ok: false,
+		error: {
+			failure: {
+				reason: "web-search-credential-rejected",
+				classification: "local:tools:mcp-client:rpc:-32001:http:403",
+			},
+		},
+	});
+	expect(closed).toBe(1);
+	expect(logs).toHaveLength(1);
+	expect(logs[0]).toStartWith("chopin: web_search unavailable at tools - ");
+	expect(logs[0]).toContain("OAuth app access token (gho_)");
+});
+
+it("reports a connection that is accepted without web_search as not offered", async () => {
+	let logs: string[] = [];
+	let result = await webSearchTool({
+		...credential(),
+		token: "ghu_abcdefghijklmnopqrstuvwxyz0123456789",
+	}, {
+		log: line => logs.push(line),
+		createClient: async () => ({
+			tools: async () => ({
+				list_pull_requests: tool({ inputSchema: z.object({}), execute: async () => ({}) }),
+			}),
+			close: async () => {},
+		}),
+	});
+	expect(result).toMatchObject({
+		ok: false,
+		error: {
+			kind: "MissingTool",
+			name: "web_search",
+			failure: {
+				reason: "web-search-not-offered",
+				classification: "local:tools:missing-tool",
+			},
+		},
+	});
+	expect(logs).toEqual([
+		expect.stringMatching(
+			/^chopin: web_search unavailable at tools - local:tools:missing-tool in \d+ms; credential: GitHub App user access token \(ghu_\); the endpoint answered but did not offer web_search; see Troubleshooting in docs\/self-hosting\.md$/,
+		),
+	]);
+});
+
+it("keeps other setup failures unavailable and says what kind of token was used", async () => {
+	let logs: string[] = [];
+	for (
+		let cause of [
+			new MCPClientError({ message: "bad gateway", statusCode: 502 }),
+			new MCPClientError({ message: "no such method", code: -32601 }),
+			Object.assign(new Error("Unable to connect"), { code: "ConnectionRefused" }),
+			"TLS handshake failed for opaque-token-value",
+		]
+	) {
+		let result = await webSearchTool({ ...credential(), token: "opaque-token-value" }, {
+			log: line => logs.push(line),
+			createClient: async () => {
+				throw cause;
+			},
+		});
+		expect(result).toMatchObject({
+			ok: false,
+			error: { failure: { reason: "web-search-unavailable" } },
+		});
+		if (result.ok) throw new Error("fixture failed");
+		expect(executionDiagnostic({ webFailure: result.error.failure.classification }))
+			.toBeDefined();
+	}
+	expect(logs.map(line => /- (\S+) in /.exec(line)?.[1])).toEqual([
+		"local:connect:mcp-client:http:502",
+		"local:connect:mcp-client:rpc:-32601",
+		"local:connect:error",
+		"local:connect:unknown",
+	]);
+	for (let line of logs) {
+		expect(line).toContain("credential: token of an unrecognized type");
+		expect(line).not.toContain("Troubleshooting");
+	}
+	expect(logs[3]).toContain("thrown string: TLS handshake failed for [redacted]");
+	expect(logs[3]).not.toContain("opaque-token-value");
+});
+
+it("masks credentials and clips a setup failure before it is logged", async () => {
+	let secret = "opaque-credential-value";
+	let message = [
+		`MCP HTTP Transport Error: POSTing to endpoint (HTTP 401): rejected echo ${secret}`,
+		`Authorization: Bearer ${secret}`,
+		"retry with github_pat_11ABCDEFG0123456789_abcdefghijklmnop or ghs_abcdefghijklmnopqrstuvwxyz0123456789",
+		`\u001b[31mcolour\u001b[0m ${secret} ${"x".repeat(900)}`,
+	].join("\n");
+	for (let name of [`Bearer ${secret}`, "ghu_abcdefghijklmnopqrstuvwxyz0123456789"]) {
+		let logs: string[] = [];
+		await webSearchTool({ ...credential(), token: secret }, {
+			log: line => logs.push(line),
+			createClient: async () => {
+				throw Object.assign(new MCPClientError({ message, statusCode: 401 }), { name });
+			},
+		});
+		expect(logs).toHaveLength(1);
+		let line = logs[0]!;
+		expect(line).not.toContain(secret);
+		expect(line).not.toContain("github_pat_");
+		expect(line).not.toContain("ghs_");
+		expect(line).not.toContain("ghu_abcdefghijkl");
+		expect(line).not.toContain("abcdefghijkl");
+		expect(line).not.toMatch(/\p{Cc}/u);
+		expect(line).toContain(
+			"rejected echo [redacted] Authorization: Bearer [redacted] retry with [redacted] or [redacted]",
+		);
+		expect(line).toMatch(/; (?:Error|\[redacted\]): MCP HTTP Transport Error: POSTing/);
+		expect(line.length).toBeLessThan(500);
+	}
+});
+
 it("does not count a resolved MCP error as a successful search", async () => {
 	let results: unknown[] = [];
 	let owner = credential();

apps/server/src/harness/web-search.ts

@@ -35,6 +35,90 @@ export type WebSearchFailureDiagnostic = {
 	webFailureToolCallId?: string;
 };
 
+export type WebSearchSetupFailure = {
+	/** The reason the research request fails with, and the card explains. */
+	reason: "web-search-credential-rejected" | "web-search-not-offered" | "web-search-unavailable";
+	/** Token-safe, for example `local:connect:mcp-client:http:401`. */
+	classification: string;
+	durationMs: number;
+};
+
+export type WebSearchSetupError = GitHubToolsError & { failure: WebSearchSetupFailure };
+
+const CREDENTIAL_TYPES: Array<[prefix: string, name: string]> = [
+	["ghu_", "GitHub App user access token"],
+	["gho_", "OAuth app access token"],
+];
+const SECRET_TEXT = /\b(?:gh[pousr]_|github_pat_)[A-Za-z0-9_]+|\b(Bearer\s+)\S+/gi;
+
+function credentialType(token: string): string {
+	let known = CREDENTIAL_TYPES.find(([prefix]) => token.startsWith(prefix));
+	return known ? `${known[1]} (${known[0]})` : "token of an unrecognized type";
+}
+
+function mcpCodes(error: unknown): string {
+	if (!MCPClientError.isInstance(error)) return "";
+	let codes = "";
+	if (Number.isSafeInteger(error.code)) codes += `:rpc:${error.code}`;
+	if (
+		Number.isSafeInteger(error.statusCode) && error.statusCode! >= 100 && error.statusCode! <= 599
+	) codes += `:http:${error.statusCode}`;
+	return codes;
+}
+
+/** One masked, clipped line of a setup error. No query has been sent when setup fails. */
+function setupMessage(error: unknown, token: string): string {
+	let mask = (text: string) => {
+		if (token.length >= 8) text = text.replaceAll(token, "[redacted]");
+		return text.replace(SECRET_TEXT, (_match, bearer?: string) => `${bearer ?? ""}[redacted]`)
+			.replace(/[\s\p{Cc}]+/gu, " ")
+			.trim();
+	};
+	if (typeof error === "string") return `thrown string: ${mask(error).slice(0, 160)}`;
+	if (!(error instanceof Error)) {
+		return `a ${error === null ? "null" : typeof error} value that is not an Error was thrown`;
+	}
+	let name = /^[A-Za-z][A-Za-z0-9_.]{0,63}$/.test(error.name) ? mask(error.name) : "Error";
+	return `${name}: ${mask(error.message).slice(0, 160)}`;
+}
+
+function setupFailure(
+	step: "connect" | "tools",
+	error: GitHubToolsError,
+	context: { token: string; began: number; log: (line: string) => void },
+): WebSearchSetupError {
+	let cause = error.kind === "Unavailable" ? error.cause : undefined;
+	let refused = MCPClientError.isInstance(cause)
+		&& (cause.statusCode === 401 || cause.statusCode === 403);
+	let kind = MCPClientError.isInstance(cause)
+		? "mcp-client"
+		: cause instanceof Error
+		? "error"
+		: "unknown";
+	let failure: WebSearchSetupFailure = {
+		reason: error.kind === "MissingTool"
+			? "web-search-not-offered"
+			: refused
+			? "web-search-credential-rejected"
+			: "web-search-unavailable",
+		classification: error.kind === "MissingTool"
+			? `local:${step}:missing-tool`
+			: `local:${step}:${kind}${mcpCodes(cause)}`,
+		durationMs: Math.round(performance.now() - context.began),
+	};
+	context.log(
+		`chopin: web_search unavailable at ${step} - ${failure.classification} in ${failure.durationMs}ms; `
+			+ `credential: ${credentialType(context.token)}; `
+			+ (error.kind === "MissingTool"
+				? "the endpoint answered but did not offer web_search"
+				: setupMessage(cause, context.token))
+			+ (failure.reason === "web-search-unavailable"
+				? ""
+				: "; see Troubleshooting in docs/self-hosting.md"),
+	);
+	return { ...error, failure };
+}
+
 /** Calls one evidence attempt may start; the upstream tool takes about two minutes each. */
 export const WEB_SEARCH_MAX_CALLS = 4;
 /** Below this much remaining attempt time a new call cannot finish usefully. */
@@ -81,7 +165,9 @@ export async function webSearchTool(
 		now?: () => number;
 		log?: (line: string) => void;
 	} = {},
-): Promise<Result<{ tool: ToolSet["web_search"]; close: () => Promise<void> }, GitHubToolsError>> {
+): Promise<
+	Result<{ tool: ToolSet["web_search"]; close: () => Promise<void> }, WebSearchSetupError>
+> {
 	// The upstream tool runs an AI research agent, so a valid response can take over a minute.
 	let timeoutMs = deps.timeoutMs ?? 180_000;
 	let now = deps.now ?? Date.now;
@@ -91,6 +177,7 @@ export async function webSearchTool(
 		throw new Error("Invalid web search timeout");
 	}
 	let client: Client;
+	let context = { token: credential.token, began: performance.now(), log };
 	try {
 		client = await (deps.createClient ?? (createMCPClient as CreateClient))({
 			transport: {
@@ -104,14 +191,14 @@ export async function webSearchTool(
 			},
 		});
 	} catch (cause) {
-		return { ok: false, error: { kind: "Unavailable", cause } };
+		return { ok: false, error: setupFailure("connect", { kind: "Unavailable", cause }, context) };
 	}
 	try {
 		let raw = await client.tools({ schemas: WEB_SEARCH_SCHEMA });
 		let picked = requireTools(raw, ["web_search"]);
 		if (!picked.ok) {
 			await client.close();
-			return picked;
+			return { ok: false, error: setupFailure("tools", picked.error, context) };
 		}
 		let source = picked.value.web_search!;
 		if (!source.execute) throw new Error("MCP web_search has no execution handler");
@@ -209,20 +296,9 @@ export async function webSearchTool(
 									? "error"
 									: "unknown";
 								let diagnostic: WebSearchFailureDiagnostic = {
-									webFailure: `local:${phase}:${kind}`,
+									webFailure: `local:${phase}:${kind}${mcpCodes(error)}`,
 									webFailureDurationMs: Math.round(performance.now() - began),
 								};
-								if (MCPClientError.isInstance(error)) {
-									if (Number.isSafeInteger(error.code)) {
-										diagnostic.webFailure += `:rpc:${error.code}`;
-									}
-									if (
-										Number.isSafeInteger(error.statusCode) && error.statusCode! >= 100
-										&& error.statusCode! <= 599
-									) {
-										diagnostic.webFailure += `:http:${error.statusCode}`;
-									}
-								}
 								if (
 									/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
 										options.toolCallId,
@@ -246,6 +322,6 @@ export async function webSearchTool(
 		};
 	} catch (cause) {
 		await client.close().catch(() => {});
-		return { ok: false, error: { kind: "Unavailable", cause } };
+		return { ok: false, error: setupFailure("tools", { kind: "Unavailable", cause }, context) };
 	}
 }