mirror of
https://github.com/open-webui/open-webui.git
synced 2026-09-27 01:40:33 -04:00
fix: pass MCP tool images to the model, not only to the UI (#30358)
When an MCP tool returns an image (e.g. a Home Assistant camera snapshot), the snapshot shows up in the tool call section but the model never sees it: it answers that there is no image. Only images arriving as inline data URIs were attached to the model request; MCP images are uploaded to Files first and their file URL was treated as display-only. Now an image file item with a file URL is attached to the model request as an input_image part in addition to staying in the tool call's displayed files. The existing URL-to-base64 step already resolves file URLs, so the model receives the image bytes; verified on a running instance with a mock MCP server and a mock upstream (the second upstream request carries the byte-identical JPEG). Inline data-URI images keep their existing model-only handling. Fixes #30327
This commit is contained in:
@@ -3487,6 +3487,8 @@ async def drain_approved_tool_calls(request, form_data, user, model, metadata) -
|
||||
output_parts.append({'type': 'input_image', 'image_url': image_url})
|
||||
else:
|
||||
display_files.append(file_item)
|
||||
if file_item.get('type') == 'image' and file_item.get('url'):
|
||||
output_parts.append({'type': 'input_image', 'image_url': file_item['url']})
|
||||
|
||||
output.append(
|
||||
{
|
||||
@@ -5858,7 +5860,7 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
frontend_output = []
|
||||
for item in full_output() if snapshot else full_output()[output_start:]:
|
||||
if item.get('type') == 'function_call_output':
|
||||
# input_image parts are base64 data URIs only for the LLM, via convert_output_to_messages
|
||||
# input_image parts are for the LLM only, via convert_output_to_messages
|
||||
item = {
|
||||
**item,
|
||||
'output': [
|
||||
@@ -6127,8 +6129,7 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
)
|
||||
result_status_by_call_id[result.get('tool_call_id', '')] = local_output_status
|
||||
|
||||
# Separate image data URIs (for LLM via input_image) from
|
||||
# other files (for frontend display via files attribute).
|
||||
# Data-URI images: LLM only. File-URL images: LLM and frontend. Other files: frontend.
|
||||
display_files = []
|
||||
for file_item in result.get('files', []):
|
||||
if file_item.get('type') == 'image' and file_item.get('url', '').startswith('data:'):
|
||||
@@ -6136,8 +6137,9 @@ async def streaming_chat_response_handler(response, ctx):
|
||||
image_url = await store_tool_result_image(request, file_item['url'], metadata, user)
|
||||
output_parts.append({'type': 'input_image', 'image_url': image_url})
|
||||
else:
|
||||
# Frontend display (MCP images, audio, etc.)
|
||||
display_files.append(file_item)
|
||||
if file_item.get('type') == 'image' and file_item.get('url'):
|
||||
output_parts.append({'type': 'input_image', 'image_url': file_item['url']})
|
||||
|
||||
output.append(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user