From 744ce6cbfec59ac835783c493ba8d5700f1b8a26 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Thu, 24 Sep 2026 05:50:08 +0200 Subject: [PATCH] fix: send the configured USER_AGENT on the Attach Webpage pre-check (#30385) Attach Webpage fails with 403 Forbidden on sites that reject the bare aiohttp user agent, Wikipedia among them, even when USER_AGENT is set. The web loader sends USER_AGENT, but the request that runs first to decide whether the URL is a page or a file does not, so the attachment fails before the loader is ever reached. The pre-check now sends USER_AGENT as the request User-Agent when it is set. With it unset the request is unchanged and keeps the aiohttp default. Verified against the real _fetch_url with https://en.wikipedia.org/wiki/OpenAI: 403 before, page detected after; USER_AGENT unset still returns the same 403 as before, and a direct PDF URL is still detected as a file. Fixes #29617 --- backend/open_webui/routers/retrieval.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index c36b0b8a53..6c5f32ccb0 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -59,6 +59,7 @@ from open_webui.env import ( SENTENCE_TRANSFORMERS_CROSS_ENCODER_SIGMOID_ACTIVATION_FUNCTION, SENTENCE_TRANSFORMERS_MODEL_KWARGS, USE_SLIM, + USER_AGENT, ) from open_webui.events import EVENTS, publish_event from open_webui.internal.db import get_async_db, get_async_session @@ -2251,9 +2252,11 @@ async def _fetch_url(url: str, max_size_mb: int | str | None) -> dict: except (TypeError, ValueError): max_bytes = None + headers = {'User-Agent': USER_AGENT} if USER_AGENT else None + async with get_ssrf_safe_session() as session: async with session.get( - url, ssl=AIOHTTP_CLIENT_SESSION_SSL, allow_redirects=AIOHTTP_CLIENT_ALLOW_REDIRECTS + url, headers=headers, ssl=AIOHTTP_CLIENT_SESSION_SSL, allow_redirects=AIOHTTP_CLIENT_ALLOW_REDIRECTS ) as response: response.raise_for_status()