From 6f6792d4848bd47be6abd6860f594e643f9de4ba Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:34:57 +0200 Subject: [PATCH] fix: pass MCP tool images to the model, not only to the UI (#30358) When an MCP tool returns an image (e.g. a Home Assistant camera snapshot), the snapshot shows up in the tool call section but the model never sees it: it answers that there is no image. Only images arriving as inline data URIs were attached to the model request; MCP images are uploaded to Files first and their file URL was treated as display-only. Now an image file item with a file URL is attached to the model request as an input_image part in addition to staying in the tool call's displayed files. The existing URL-to-base64 step already resolves file URLs, so the model receives the image bytes; verified on a running instance with a mock MCP server and a mock upstream (the second upstream request carries the byte-identical JPEG). Inline data-URI images keep their existing model-only handling. Fixes #30327 --- backend/open_webui/utils/middleware.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 82346043b4..0cb0f2b6d9 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -3487,6 +3487,8 @@ async def drain_approved_tool_calls(request, form_data, user, model, metadata) - output_parts.append({'type': 'input_image', 'image_url': image_url}) else: display_files.append(file_item) + if file_item.get('type') == 'image' and file_item.get('url'): + output_parts.append({'type': 'input_image', 'image_url': file_item['url']}) output.append( { @@ -5858,7 +5860,7 @@ async def streaming_chat_response_handler(response, ctx): frontend_output = [] for item in full_output() if snapshot else full_output()[output_start:]: if item.get('type') == 'function_call_output': - # input_image parts are base64 data URIs only for the LLM, via convert_output_to_messages + # input_image parts are for the LLM only, via convert_output_to_messages item = { **item, 'output': [ @@ -6127,8 +6129,7 @@ async def streaming_chat_response_handler(response, ctx): ) result_status_by_call_id[result.get('tool_call_id', '')] = local_output_status - # Separate image data URIs (for LLM via input_image) from - # other files (for frontend display via files attribute). + # Data-URI images: LLM only. File-URL images: LLM and frontend. Other files: frontend. display_files = [] for file_item in result.get('files', []): if file_item.get('type') == 'image' and file_item.get('url', '').startswith('data:'): @@ -6136,8 +6137,9 @@ async def streaming_chat_response_handler(response, ctx): image_url = await store_tool_result_image(request, file_item['url'], metadata, user) output_parts.append({'type': 'input_image', 'image_url': image_url}) else: - # Frontend display (MCP images, audio, etc.) display_files.append(file_item) + if file_item.get('type') == 'image' and file_item.get('url'): + output_parts.append({'type': 'input_image', 'image_url': file_item['url']}) output.append( {