diff --git a/.env.example b/.env.example index 22b128ce8f..313024511a 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,9 @@ ENABLE_RAG_CSV_SUMMARY=false # Set to true to preserve backing file records, storage blobs, and per-file vectors when files are removed from knowledge bases. ENABLE_KNOWLEDGE_FILE_RETENTION=false +# Comma-separated chunk metadata keys to expose to the model alongside retrieved content. +RAG_SOURCE_METADATA_KEYS='' + # Set to false to disable workspace Tools and Functions. ENABLE_PLUGINS=true diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index 9179f9a4b8..68c4272872 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -371,6 +371,9 @@ ENABLE_QUERIES_CACHE = os.getenv('ENABLE_QUERIES_CACHE', 'False').lower() == 'tr ENABLE_ADMIN_CHAT_ACCESS = os.getenv('ENABLE_ADMIN_CHAT_ACCESS', 'True').lower() == 'true' RAG_SYSTEM_CONTEXT = os.getenv('RAG_SYSTEM_CONTEXT', 'False').lower() == 'true' +# Empty by default: chunk metadata also holds internal bookkeeping (file hashes, collection names, scores). +RAG_SOURCE_METADATA_KEYS = [key.strip() for key in os.getenv('RAG_SOURCE_METADATA_KEYS', '').split(',') if key.strip()] + #################################### # REDIS #################################### diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index 531b2605e2..055f750122 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -34,6 +34,7 @@ from open_webui.env import ( ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS, MPS_INFERENCE_LOCK, OFFLINE_MODE, + RAG_SOURCE_METADATA_KEYS, USE_SLIM, ) from open_webui.models.access_grants import AccessGrants @@ -1361,6 +1362,11 @@ async def filter_accessible_collections( return validated +def filter_source_metadata(metadata: dict) -> dict: + """Keep only the chunk metadata keys the operator allowed the model to see.""" + return {key: metadata[key] for key in RAG_SOURCE_METADATA_KEYS if metadata.get(key) is not None} + + async def get_sources_from_items( request, items, diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index 36896f9a2e..f37e0d273f 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -29,7 +29,7 @@ from open_webui.models.memories import Memories from open_webui.models.messages import Message, Messages from open_webui.models.notes import Notes from open_webui.models.users import UserModel -from open_webui.retrieval.utils import get_content_from_url +from open_webui.retrieval.utils import filter_source_metadata, get_content_from_url from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT from open_webui.routers.images import ( CreateImageForm, @@ -2610,6 +2610,7 @@ async def query_chat_files( for idx, doc in enumerate(documents): metadata = metadatas[idx] if idx < len(metadatas) and isinstance(metadatas[idx], dict) else {} chunk = { + **filter_source_metadata(metadata), 'content': doc, 'source': metadata.get('source', metadata.get('name', source_info.get('name', 'Unknown'))), 'file_id': metadata.get('file_id', source_info.get('id', '')), @@ -3322,6 +3323,7 @@ async def query_knowledge_files( for idx, doc in enumerate(documents): chunk_info = { + **filter_source_metadata(metadatas[idx]), 'content': doc, 'source': metadatas[idx].get('source', metadatas[idx].get('name', 'Unknown')), 'file_id': metadatas[idx].get('file_id', ''), @@ -3345,6 +3347,7 @@ async def query_knowledge_files( for idx, doc in enumerate(documents): metadata = metadatas[idx] if idx < len(metadatas) else {} chunk_info = { + **filter_source_metadata(metadata), 'content': doc, 'source': metadata.get('source', metadata.get('name', knowledge.name)), 'file_id': metadata.get('file_id', f'external-{knowledge.id}'), diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 421fdaa0db..cf4f001d71 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -2,6 +2,7 @@ import ast import asyncio import base64 import copy +import html import inspect import json import logging @@ -52,7 +53,7 @@ from open_webui.models.models import Models from open_webui.models.notes import Notes from open_webui.models.oauth_sessions import OAuthSessions from open_webui.models.users import UserModel, Users -from open_webui.retrieval.utils import get_sources_from_items +from open_webui.retrieval.utils import filter_source_metadata, get_sources_from_items from open_webui.routers.images import ( CreateImageForm, EditImageForm, @@ -952,11 +953,17 @@ def get_source_context(sources: list, source_ids: dict = None, include_content: src_type = source.get('source', {}).get('type') src_rid = source.get('source', {}).get('id') body = doc if include_content else '' + extra_attrs = '' + for key, value in filter_source_metadata(meta).items(): + if key in ('id', 'name', 'resource-type', 'resource-id'): + continue + extra_attrs += f' {key}="{html.escape(str(value))}"' context_string += ( f'{body}\n' ) return context_string