fix(runtime): bound OpenAI Chat image tool results

packages/runtime-host/src/__tests__/execution-model-composition.test.ts

@@ -938,7 +938,7 @@ test('backend creation admits an enabled model a snapshot never listed', async (
   await backend.dispose();
 });
 
-test('Host reopens one projected image from its ArtifactStore authority', async () => {
+test('Host keeps OpenAI Chat projected image results bounded across replay', async () => {
   const base = await mkdtemp(join(tmpdir(), 'maka-host-projection-image-'));
   const capability = await resolveStorageRoot({
     path: join(base, 'interactive'),
@@ -969,16 +969,15 @@ test('Host reopens one projected image from its ArtifactStore authority', async
   if (!owner) return;
   const provider = await startProvider();
   provider.configureProjectionImageFlow('ProjectedImage');
-  const assertProjectedImage = (body: Record<string, unknown> | undefined) => {
+  const assertProjectedImageFallback = (body: Record<string, unknown> | undefined) => {
     assert.ok(body);
-    assert.doesNotMatch(JSON.stringify(body), /raw execution fact/u);
-    assert.deepEqual(JSON.parse(latestToolResultText(body) ?? 'null'), [
-      {
-        type: 'file',
-        mediaType: 'image/png',
-        data: { type: 'data', data: pngBytes.toString('base64') },
-      },
-    ]);
+    const serializedBody = JSON.stringify(body);
+    assert.doesNotMatch(serializedBody, /raw execution fact/u);
+    assert.doesNotMatch(serializedBody, new RegExp(pngBytes.toString('base64'), 'u'));
+    assert.equal(
+      latestToolResultText(body),
+      'Image was read successfully, but this provider protocol cannot represent image content in a tool result. The binary image was omitted.',
+    );
   };
   let backend: Awaited<ReturnType<typeof createHostAiSdkBackend>> | undefined;
   let artifacts: Awaited<ReturnType<typeof openInteractiveArtifactStoreForWrite>> | undefined;
@@ -1030,7 +1029,7 @@ test('Host reopens one projected image from its ArtifactStore authority', async
     }
     const liveRequests = provider.requests.filter((request) => request.body.stream === true);
     assert.equal(liveRequests.length, 2);
-    assertProjectedImage(liveRequests[1]?.body);
+    assertProjectedImageFallback(liveRequests[1]?.body);
 
     const nextRunId = 'projection-image-next-run';
     const nextText = 'Continue in the same process.';
@@ -1061,7 +1060,7 @@ test('Host reopens one projected image from its ArtifactStore authority', async
     }
     const nextTurnRequests = provider.requests.filter((request) => request.body.stream === true);
     assert.equal(nextTurnRequests.length, 3);
-    assertProjectedImage(nextTurnRequests[2]?.body);
+    assertProjectedImageFallback(nextTurnRequests[2]?.body);
 
     await backend.dispose();
     backend = undefined;
@@ -1097,7 +1096,7 @@ test('Host reopens one projected image from its ArtifactStore authority', async
     }
     const streamRequests = provider.requests.filter((request) => request.body.stream === true);
     assert.equal(streamRequests.length, 4);
-    assertProjectedImage(streamRequests[3]?.body);
+    assertProjectedImageFallback(streamRequests[3]?.body);
   } finally {
     await backend?.dispose();
     artifacts?.close();

packages/runtime/src/__tests__/ai-sdk-backend.test.ts

@@ -2694,6 +2694,117 @@ describe('AiSdkBackend model history', () => {
     );
   });
 
+  test('OpenAI-compatible Chat omits image bytes from string-valued tool content', async () => {
+    const imageBytes = new TextEncoder().encode('MAKA_IMAGE_TOOL_RESULT');
+    const imageBase64 = Buffer.from(imageBytes).toString('base64');
+    let readCount = 0;
+    let requestBody: Record<string, unknown> | undefined;
+    const fetch = (async (_url: string | URL | Request, init?: RequestInit) => {
+      requestBody = JSON.parse(String(init?.body)) as Record<string, unknown>;
+      const chunks = [
+        {
+          id: 'chatcmpl-image-tool-result',
+          object: 'chat.completion.chunk',
+          created: 1,
+          model: 'relay-model',
+          choices: [{ index: 0, delta: { role: 'assistant', content: 'ok' }, finish_reason: null }],
+        },
+        {
+          id: 'chatcmpl-image-tool-result',
+          object: 'chat.completion.chunk',
+          created: 1,
+          model: 'relay-model',
+          choices: [{ index: 0, delta: {}, finish_reason: 'stop' }],
+          usage: { prompt_tokens: 12, completion_tokens: 1, total_tokens: 13 },
+        },
+      ];
+      return new Response(
+        `${chunks.map((chunk) => `data: ${JSON.stringify(chunk)}`).join('\n\n')}\n\ndata: [DONE]\n\n`,
+        { status: 200, headers: { 'content-type': 'text/event-stream' } },
+      );
+    }) as unknown as typeof globalThis.fetch;
+    const backend = createBackend({
+      connection: {
+        slug: 'relay',
+        providerType: 'openai-compatible',
+        baseUrl: 'https://relay.example/v1',
+        defaultModel: 'relay-model',
+      },
+      apiKey: 'relay-token',
+      modelId: 'relay-model',
+      modelFactory: (input) => getAIModel({ ...input, fetch }),
+      tools: [],
+      supportsVision: true,
+      readAttachmentBytes: async () => {
+        readCount += 1;
+        return { ok: true, bytes: imageBytes };
+      },
+    });
+
+    await drain(
+      backend.send({
+        turnId: 'turn-current',
+        text: 'continue',
+        context: [],
+        ...sameRouteReplayProvenance('relay-model'),
+        runtimeContext: [
+          runtimeTextEvent({
+            id: 'rt-user',
+            turnId: 'turn-prev',
+            role: 'user',
+            author: 'user',
+            text: 'read the image',
+          }),
+          runtimeEvent({
+            id: 'rt-call',
+            turnId: 'turn-prev',
+            role: 'model',
+            author: 'agent',
+            content: {
+              kind: 'function_call',
+              id: 'tool-image',
+              name: 'Read',
+              args: { path: 'chart.png' },
+            },
+          }),
+          runtimeEvent({
+            id: 'rt-result',
+            turnId: 'turn-prev',
+            role: 'tool',
+            author: 'tool',
+            content: {
+              kind: 'function_response',
+              id: 'tool-image',
+              name: 'Read',
+              isError: false,
+              result: {
+                kind: 'image',
+                mimeType: 'image/png',
+                ref: {
+                  kind: 'session_file',
+                  sessionId: 'session-1',
+                  relativePath: 'chart.png',
+                },
+              },
+            },
+          }),
+        ],
+      }),
+    );
+
+    const messages = requestBody?.messages;
+    assert.ok(Array.isArray(messages));
+    const assistant = messages.find((message) => message?.role === 'assistant');
+    const tool = messages.find((message) => message?.role === 'tool');
+    assert.ok(assistant && Array.isArray(assistant.tool_calls));
+    assert.equal(assistant.tool_calls[0]?.id, 'tool-image');
+    assert.equal(tool?.tool_call_id, 'tool-image');
+    assert.equal(typeof tool?.content, 'string');
+    assert.match(tool.content, /cannot represent image content in a tool result/);
+    assert.equal(JSON.stringify(requestBody).includes(imageBase64), false);
+    assert.equal(readCount, 0, 'unsupported tool-result media must not load image bytes');
+  });
+
   test('budgets replayed image tool results by durable occurrence instead of reused tool-call ids', async () => {
     const bytes = new Uint8Array(10);
     const model = completionModel();

packages/runtime/src/ai-sdk-message-projection.ts

@@ -122,6 +122,13 @@ function toolResultText(text: string): ToolResultOutput {
   return { type: 'content', value: [{ type: 'text', text }] };
 }
 
+const UNSUPPORTED_IMAGE_TOOL_RESULT_MESSAGE =
+  'Image was read successfully, but this provider protocol cannot represent image content in a tool result. The binary image was omitted.';
+
+function plainToolResultText(text: string): ToolResultOutput {
+  return { type: 'text', value: text };
+}
+
 function nativeApplyPatchFailureOutput(output: ToolResultOutput): ToolResultOutput {
   const value = output.type === 'json' || output.type === 'error-json' ? output.value : undefined;
   const record = value && typeof value === 'object' && !Array.isArray(value) ? value : undefined;
@@ -704,6 +711,9 @@ export class AiSdkMessageProjection {
     decisionKey: string,
   ): Promise<ToolResultOutput> {
     if (isError || !isImageToolResult(output)) return toolResultOutput(output, isError);
+    if (!this.input.modelAdapter.supportsImageToolResults()) {
+      return plainToolResultText(UNSUPPORTED_IMAGE_TOOL_RESULT_MESSAGE);
+    }
     if (this.input.supportsVision !== true) {
       return toolResultText('Image was read, but the selected model does not support image input.');
     }
@@ -747,6 +757,16 @@ export class AiSdkMessageProjection {
     decisionKey: string,
   ): Promise<ToolResultOutput> {
     if (projection.kind !== 'content') return durableProjectionToToolResultOutput(projection);
+    if (!this.input.modelAdapter.supportsImageToolResults()) {
+      const text = projection.parts
+        .filter((part): part is Extract<typeof part, { kind: 'text' }> => part.kind === 'text')
+        .map((part) => part.text)
+        .filter((part) => part.length > 0);
+      if (projection.parts.some((part) => part.kind === 'artifact')) {
+        text.push(UNSUPPORTED_IMAGE_TOOL_RESULT_MESSAGE);
+      }
+      return plainToolResultText(text.join('\n'));
+    }
     const value: Extract<ToolResultOutput, { type: 'content' }>['value'] = [];
     for (const [index, part] of projection.parts.entries()) {
       if (part.kind === 'text') {

packages/runtime/src/model-adapter.ts

@@ -191,6 +191,13 @@ export class ModelAdapter {
     };
   }
 
+  supportsImageToolResults(): boolean {
+    // Chat Completions only accepts string content for `role: tool`. Passing a
+    // content-shaped image result makes the SDK stringify its file part,
+    // turning base64 into ordinary model-visible text.
+    return this.runtime.wire !== 'openai-chat';
+  }
+
   resolveModel(): unknown {
     if (providerAuthRequiresSecret(this.input.connection.providerType) && !this.input.apiKey) {
       throw new Error(`No API key stored for connection "${this.input.connection.slug}"`);