diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index c36b0b8a53..6c5f32ccb0 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -59,6 +59,7 @@ from open_webui.env import ( SENTENCE_TRANSFORMERS_CROSS_ENCODER_SIGMOID_ACTIVATION_FUNCTION, SENTENCE_TRANSFORMERS_MODEL_KWARGS, USE_SLIM, + USER_AGENT, ) from open_webui.events import EVENTS, publish_event from open_webui.internal.db import get_async_db, get_async_session @@ -2251,9 +2252,11 @@ async def _fetch_url(url: str, max_size_mb: int | str | None) -> dict: except (TypeError, ValueError): max_bytes = None + headers = {'User-Agent': USER_AGENT} if USER_AGENT else None + async with get_ssrf_safe_session() as session: async with session.get( - url, ssl=AIOHTTP_CLIENT_SESSION_SSL, allow_redirects=AIOHTTP_CLIENT_ALLOW_REDIRECTS + url, headers=headers, ssl=AIOHTTP_CLIENT_SESSION_SSL, allow_redirects=AIOHTTP_CLIENT_ALLOW_REDIRECTS ) as response: response.raise_for_status()