diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index 39823f8b8d..602a80197d 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -938,7 +938,7 @@ test('backend creation admits an enabled model a snapshot never listed', async ( await backend.dispose(); }); -test('Host reopens one projected image from its ArtifactStore authority', async () => { +test('Host keeps OpenAI Chat projected image results bounded across replay', async () => { const base = await mkdtemp(join(tmpdir(), 'maka-host-projection-image-')); const capability = await resolveStorageRoot({ path: join(base, 'interactive'), @@ -969,16 +969,15 @@ test('Host reopens one projected image from its ArtifactStore authority', async if (!owner) return; const provider = await startProvider(); provider.configureProjectionImageFlow('ProjectedImage'); - const assertProjectedImage = (body: Record | undefined) => { + const assertProjectedImageFallback = (body: Record | undefined) => { assert.ok(body); - assert.doesNotMatch(JSON.stringify(body), /raw execution fact/u); - assert.deepEqual(JSON.parse(latestToolResultText(body) ?? 'null'), [ - { - type: 'file', - mediaType: 'image/png', - data: { type: 'data', data: pngBytes.toString('base64') }, - }, - ]); + const serializedBody = JSON.stringify(body); + assert.doesNotMatch(serializedBody, /raw execution fact/u); + assert.doesNotMatch(serializedBody, new RegExp(pngBytes.toString('base64'), 'u')); + assert.equal( + latestToolResultText(body), + 'Image was read successfully, but this provider protocol cannot represent image content in a tool result. The binary image was omitted.', + ); }; let backend: Awaited> | undefined; let artifacts: Awaited> | undefined; @@ -1030,7 +1029,7 @@ test('Host reopens one projected image from its ArtifactStore authority', async } const liveRequests = provider.requests.filter((request) => request.body.stream === true); assert.equal(liveRequests.length, 2); - assertProjectedImage(liveRequests[1]?.body); + assertProjectedImageFallback(liveRequests[1]?.body); const nextRunId = 'projection-image-next-run'; const nextText = 'Continue in the same process.'; @@ -1061,7 +1060,7 @@ test('Host reopens one projected image from its ArtifactStore authority', async } const nextTurnRequests = provider.requests.filter((request) => request.body.stream === true); assert.equal(nextTurnRequests.length, 3); - assertProjectedImage(nextTurnRequests[2]?.body); + assertProjectedImageFallback(nextTurnRequests[2]?.body); await backend.dispose(); backend = undefined; @@ -1097,7 +1096,7 @@ test('Host reopens one projected image from its ArtifactStore authority', async } const streamRequests = provider.requests.filter((request) => request.body.stream === true); assert.equal(streamRequests.length, 4); - assertProjectedImage(streamRequests[3]?.body); + assertProjectedImageFallback(streamRequests[3]?.body); } finally { await backend?.dispose(); artifacts?.close(); diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index 4b19e3308a..f1ce46faa3 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -2694,6 +2694,117 @@ describe('AiSdkBackend model history', () => { ); }); + test('OpenAI-compatible Chat omits image bytes from string-valued tool content', async () => { + const imageBytes = new TextEncoder().encode('MAKA_IMAGE_TOOL_RESULT'); + const imageBase64 = Buffer.from(imageBytes).toString('base64'); + let readCount = 0; + let requestBody: Record | undefined; + const fetch = (async (_url: string | URL | Request, init?: RequestInit) => { + requestBody = JSON.parse(String(init?.body)) as Record; + const chunks = [ + { + id: 'chatcmpl-image-tool-result', + object: 'chat.completion.chunk', + created: 1, + model: 'relay-model', + choices: [{ index: 0, delta: { role: 'assistant', content: 'ok' }, finish_reason: null }], + }, + { + id: 'chatcmpl-image-tool-result', + object: 'chat.completion.chunk', + created: 1, + model: 'relay-model', + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + usage: { prompt_tokens: 12, completion_tokens: 1, total_tokens: 13 }, + }, + ]; + return new Response( + `${chunks.map((chunk) => `data: ${JSON.stringify(chunk)}`).join('\n\n')}\n\ndata: [DONE]\n\n`, + { status: 200, headers: { 'content-type': 'text/event-stream' } }, + ); + }) as unknown as typeof globalThis.fetch; + const backend = createBackend({ + connection: { + slug: 'relay', + providerType: 'openai-compatible', + baseUrl: 'https://relay.example/v1', + defaultModel: 'relay-model', + }, + apiKey: 'relay-token', + modelId: 'relay-model', + modelFactory: (input) => getAIModel({ ...input, fetch }), + tools: [], + supportsVision: true, + readAttachmentBytes: async () => { + readCount += 1; + return { ok: true, bytes: imageBytes }; + }, + }); + + await drain( + backend.send({ + turnId: 'turn-current', + text: 'continue', + context: [], + ...sameRouteReplayProvenance('relay-model'), + runtimeContext: [ + runtimeTextEvent({ + id: 'rt-user', + turnId: 'turn-prev', + role: 'user', + author: 'user', + text: 'read the image', + }), + runtimeEvent({ + id: 'rt-call', + turnId: 'turn-prev', + role: 'model', + author: 'agent', + content: { + kind: 'function_call', + id: 'tool-image', + name: 'Read', + args: { path: 'chart.png' }, + }, + }), + runtimeEvent({ + id: 'rt-result', + turnId: 'turn-prev', + role: 'tool', + author: 'tool', + content: { + kind: 'function_response', + id: 'tool-image', + name: 'Read', + isError: false, + result: { + kind: 'image', + mimeType: 'image/png', + ref: { + kind: 'session_file', + sessionId: 'session-1', + relativePath: 'chart.png', + }, + }, + }, + }), + ], + }), + ); + + const messages = requestBody?.messages; + assert.ok(Array.isArray(messages)); + const assistant = messages.find((message) => message?.role === 'assistant'); + const tool = messages.find((message) => message?.role === 'tool'); + assert.ok(assistant && Array.isArray(assistant.tool_calls)); + assert.equal(assistant.tool_calls[0]?.id, 'tool-image'); + assert.equal(tool?.tool_call_id, 'tool-image'); + assert.equal(typeof tool?.content, 'string'); + assert.match(tool.content, /cannot represent image content in a tool result/); + assert.equal(JSON.stringify(requestBody).includes(imageBase64), false); + assert.equal(readCount, 0, 'unsupported tool-result media must not load image bytes'); + }); + test('budgets replayed image tool results by durable occurrence instead of reused tool-call ids', async () => { const bytes = new Uint8Array(10); const model = completionModel(); diff --git a/packages/runtime/src/ai-sdk-message-projection.ts b/packages/runtime/src/ai-sdk-message-projection.ts index 46e6efbe6a..6a58d2ee2a 100644 --- a/packages/runtime/src/ai-sdk-message-projection.ts +++ b/packages/runtime/src/ai-sdk-message-projection.ts @@ -122,6 +122,13 @@ function toolResultText(text: string): ToolResultOutput { return { type: 'content', value: [{ type: 'text', text }] }; } +const UNSUPPORTED_IMAGE_TOOL_RESULT_MESSAGE = + 'Image was read successfully, but this provider protocol cannot represent image content in a tool result. The binary image was omitted.'; + +function plainToolResultText(text: string): ToolResultOutput { + return { type: 'text', value: text }; +} + function nativeApplyPatchFailureOutput(output: ToolResultOutput): ToolResultOutput { const value = output.type === 'json' || output.type === 'error-json' ? output.value : undefined; const record = value && typeof value === 'object' && !Array.isArray(value) ? value : undefined; @@ -704,6 +711,9 @@ export class AiSdkMessageProjection { decisionKey: string, ): Promise { if (isError || !isImageToolResult(output)) return toolResultOutput(output, isError); + if (!this.input.modelAdapter.supportsImageToolResults()) { + return plainToolResultText(UNSUPPORTED_IMAGE_TOOL_RESULT_MESSAGE); + } if (this.input.supportsVision !== true) { return toolResultText('Image was read, but the selected model does not support image input.'); } @@ -747,6 +757,16 @@ export class AiSdkMessageProjection { decisionKey: string, ): Promise { if (projection.kind !== 'content') return durableProjectionToToolResultOutput(projection); + if (!this.input.modelAdapter.supportsImageToolResults()) { + const text = projection.parts + .filter((part): part is Extract => part.kind === 'text') + .map((part) => part.text) + .filter((part) => part.length > 0); + if (projection.parts.some((part) => part.kind === 'artifact')) { + text.push(UNSUPPORTED_IMAGE_TOOL_RESULT_MESSAGE); + } + return plainToolResultText(text.join('\n')); + } const value: Extract['value'] = []; for (const [index, part] of projection.parts.entries()) { if (part.kind === 'text') { diff --git a/packages/runtime/src/model-adapter.ts b/packages/runtime/src/model-adapter.ts index 29aec059e4..cb03722090 100644 --- a/packages/runtime/src/model-adapter.ts +++ b/packages/runtime/src/model-adapter.ts @@ -191,6 +191,13 @@ export class ModelAdapter { }; } + supportsImageToolResults(): boolean { + // Chat Completions only accepts string content for `role: tool`. Passing a + // content-shaped image result makes the SDK stringify its file part, + // turning base64 into ordinary model-visible text. + return this.runtime.wire !== 'openai-chat'; + } + resolveModel(): unknown { if (providerAuthRequiresSecret(this.input.connection.providerType) && !this.input.apiKey) { throw new Error(`No API key stored for connection "${this.input.connection.slug}"`);