mirror of
https://github.com/open-webui/open-webui.git
synced 2026-09-25 17:00:52 -04:00
refac
This commit is contained in:
Timothy Jaeryang Baek
@@ -2462,6 +2462,7 @@ async def grep_chat_files(
|
||||
"""
|
||||
Search exact text across files attached to the current chat.
|
||||
Pass file_id from the attached_files block to search one file.
|
||||
Auto-detected regex uses RE2 syntax; no lookarounds/backreferences, and shorthand classes are ASCII-only.
|
||||
|
||||
:param pattern: The text pattern to search for
|
||||
:param file_id: Optional attached file ID to search within a single file
|
||||
@@ -2638,6 +2639,7 @@ async def grep_knowledge_files(
|
||||
Search for exact text across knowledge files. Returns matching lines with line numbers.
|
||||
Unlike query_knowledge_files (semantic/vector search), this performs exact string matching.
|
||||
Automatically detects regex patterns (e.g. "error|warn", "version \\d+").
|
||||
Regex uses RE2 syntax; no lookarounds/backreferences, and shorthand character classes are ASCII-only.
|
||||
Helpful for literal strings, identifiers, error messages, or regex-style searches.
|
||||
|
||||
:param pattern: The text pattern to search for (regex auto-detected)
|
||||
|
||||
@@ -17,7 +17,7 @@ from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
from typing import Optional
|
||||
|
||||
import regex
|
||||
import re2
|
||||
from fastapi import Request
|
||||
|
||||
from open_webui.env import (
|
||||
@@ -31,12 +31,9 @@ log = logging.getLogger(__name__)
|
||||
DEFAULT_HEAD_LINES = 10
|
||||
DEFAULT_TAIL_LINES = 10
|
||||
|
||||
# Matching time allowed per tool call. Backtracking cost is exponential in the length of the
|
||||
# matched text, so capping the pattern or the line does not bound it.
|
||||
# Total matching time allowed per tool call, checked between RE2's linear-time searches.
|
||||
MATCH_BUDGET_SECONDS = 2.0
|
||||
MAX_REGEX_QUANTIFIER_COUNT = 2_000
|
||||
MAX_REGEX_QUANTIFIER_EXPANSION = 100_000
|
||||
_COUNTED_QUANTIFIER_RE = re.compile(r'(?<!\\)\{(\d+)(?:,\d*)?\}')
|
||||
MAX_SEARCH_PATTERN_LENGTH = 4_096
|
||||
|
||||
|
||||
class MatchBudgetExceeded(Exception):
|
||||
@@ -90,48 +87,36 @@ def normalize_regex(pattern: str) -> str:
|
||||
return pattern.replace(r'\|', '|').replace(r'\|', '|')
|
||||
|
||||
|
||||
def validate_regex_quantifiers(pattern: str) -> str | None:
|
||||
"""Reject counted quantifiers that make regex compilation expand too much."""
|
||||
quantifier_expansion = 1
|
||||
for quantifier in _COUNTED_QUANTIFIER_RE.finditer(pattern):
|
||||
count_text = quantifier.group(1)
|
||||
count = int(count_text) if len(count_text) <= 6 else MAX_REGEX_QUANTIFIER_COUNT + 1
|
||||
if count > MAX_REGEX_QUANTIFIER_COUNT:
|
||||
return f'Regex quantifier counts over {MAX_REGEX_QUANTIFIER_COUNT:g} are not supported'
|
||||
|
||||
# ponytail: conservative expansion catches nested quantifier bombs without mirroring regex syntax.
|
||||
quantifier_expansion *= max(count, 1)
|
||||
if quantifier_expansion > MAX_REGEX_QUANTIFIER_EXPANSION:
|
||||
return 'Regex quantifiers expand too much, lower the counts'
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def build_matcher(pattern: str, case_insensitive: bool = False, use_regex: bool = False) -> tuple:
|
||||
"""Build a matcher function. Returns (match_fn, error_str_or_None)."""
|
||||
if len(pattern) > MAX_SEARCH_PATTERN_LENGTH:
|
||||
return None, f'Search patterns over {MAX_SEARCH_PATTERN_LENGTH} characters are not supported'
|
||||
|
||||
if not use_regex and is_regex_pattern(pattern):
|
||||
use_regex = True
|
||||
|
||||
if use_regex:
|
||||
normalized = normalize_regex(pattern)
|
||||
quantifier_error = validate_regex_quantifiers(normalized)
|
||||
if quantifier_error:
|
||||
return None, quantifier_error
|
||||
try:
|
||||
re_flags = regex.IGNORECASE if case_insensitive else 0
|
||||
compiled = regex.compile(normalized, re_flags)
|
||||
except regex.error as e:
|
||||
return None, f'Invalid regex: {e}'
|
||||
options = re2.Options()
|
||||
options.case_sensitive = not case_insensitive
|
||||
options.max_mem = 1 << 20 # Bound compiled programs and the engine's matching cache to 1 MiB.
|
||||
options.log_errors = False
|
||||
compiled = re2.compile(normalized, options=options)
|
||||
except re2.error as e:
|
||||
return None, f'Invalid or unsupported regex (RE2 syntax): {e}'
|
||||
|
||||
budget = _active_budget.get() or MatchBudget()
|
||||
|
||||
def matches(line: str) -> bool:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
# A negative timeout disables it, so an exhausted budget must not reach search().
|
||||
if budget.remaining <= 0:
|
||||
raise TimeoutError
|
||||
return bool(compiled.search(line, timeout=budget.remaining))
|
||||
matched = bool(compiled.search(line))
|
||||
if time.monotonic() - started >= budget.remaining:
|
||||
raise TimeoutError
|
||||
return matched
|
||||
except TimeoutError:
|
||||
raise MatchBudgetExceeded(f'Search exceeded {MATCH_BUDGET_SECONDS:g}s, narrow the pattern') from None
|
||||
finally:
|
||||
@@ -1188,6 +1173,7 @@ async def kb_exec(
|
||||
|
||||
Pipes: grep "auth" | head -5
|
||||
Files: reference by path (docs/api/auth.md), filename, or file ID
|
||||
Regex: RE2 syntax; no lookarounds/backreferences. Shorthand character classes are ASCII-only.
|
||||
|
||||
:param command: A filesystem command string
|
||||
:return: Command output as text
|
||||
|
||||
@@ -19,7 +19,8 @@ authlib==1.7.2
|
||||
joserfc==1.7.4
|
||||
|
||||
requests==2.34.2
|
||||
regex==2026.5.9 # supports a per-search timeout, which `re` does not
|
||||
regex==2026.5.9
|
||||
google-re2==1.1.20251105 # bounded compilation and linear-time knowledge searches
|
||||
aiohttp==3.13.5 # do not update to 3.13.3 - broken
|
||||
aiodns==3.6.1 # keep pinned: 4.x pulls pycares 5 (c-ares 1.34.6) which breaks DNS on some hosts (#28013, #28215); opt-in via AIOHTTP_CLIENT_ASYNC_DNS_RESOLVER
|
||||
aiocache==0.12.3
|
||||
|
||||
@@ -14,7 +14,8 @@ authlib==1.7.2
|
||||
joserfc==1.7.4
|
||||
|
||||
requests==2.34.2
|
||||
regex==2026.5.9 # supports a per-search timeout, which `re` does not
|
||||
regex==2026.5.9
|
||||
google-re2==1.1.20251105 # bounded compilation and linear-time knowledge searches
|
||||
aiohttp==3.13.5 # do not update to 3.13.3 - broken
|
||||
aiodns==3.6.1 # keep pinned: 4.x pulls pycares 5 (c-ares 1.34.6) which breaks DNS on some hosts (#28013, #28215); opt-in via AIOHTTP_CLIENT_ASYNC_DNS_RESOLVER
|
||||
aiocache==0.12.3
|
||||
|
||||
@@ -50,6 +50,7 @@ dependencies = [
|
||||
|
||||
"tiktoken==0.13.0",
|
||||
"regex==2026.5.9",
|
||||
"google-re2==1.1.20251105",
|
||||
"mcp==1.27.2",
|
||||
|
||||
"openai==2.29.0",
|
||||
|
||||
@@ -87,7 +87,6 @@
|
||||
{compactPreview}
|
||||
{editCodeBlock}
|
||||
{topPadding}
|
||||
{onInsertToNote}
|
||||
/>
|
||||
{:else if (history.messages[history.messages[messageId].parentId]?.models?.length ?? 1) === 1}
|
||||
<ResponseMessage
|
||||
@@ -118,6 +117,7 @@
|
||||
{compactPreview}
|
||||
{editCodeBlock}
|
||||
{topPadding}
|
||||
{onInsertToNote}
|
||||
/>
|
||||
{:else}
|
||||
{#key messageId}
|
||||
|
||||
@@ -49,6 +49,7 @@
|
||||
import RateComment from './RateComment.svelte';
|
||||
import WebSearchResults from './ResponseMessage/WebSearchResults.svelte';
|
||||
import Sparkles from '$lib/components/icons/Sparkles.svelte';
|
||||
import ArrowUpLeft from '$lib/components/icons/ArrowUpLeft.svelte';
|
||||
|
||||
import DeleteConfirmDialog from '$lib/components/common/ConfirmDialog.svelte';
|
||||
|
||||
@@ -943,6 +944,22 @@
|
||||
class="flex items-center justify-start overflow-x-auto whitespace-nowrap buttons text-gray-600 dark:text-gray-500 mt-0.5 [&>*]:shrink-0"
|
||||
>
|
||||
{#if message.done || siblings.length > 1}
|
||||
{#if message.done && onInsertToNote && visibleResponseContent}
|
||||
<Tooltip content={$i18n.t('Insert into note')} placement="bottom">
|
||||
<button
|
||||
aria-label={$i18n.t('Insert into note')}
|
||||
class="{isLastMessage || ($settings?.highContrastMode ?? false)
|
||||
? 'visible'
|
||||
: 'hover-reveal'} p-1.5 hover:bg-black/5 dark:hover:bg-white/5 rounded-lg dark:hover:text-white hover:text-black transition"
|
||||
on:click={() => {
|
||||
onInsertToNote?.(visibleResponseContent);
|
||||
}}
|
||||
>
|
||||
<ArrowUpLeft className="size-3.5" strokeWidth="2" />
|
||||
</button>
|
||||
</Tooltip>
|
||||
{/if}
|
||||
|
||||
{#if siblings.length > 1}
|
||||
<div class="flex self-center min-w-fit" dir="ltr">
|
||||
<button
|
||||
@@ -1101,22 +1118,6 @@
|
||||
</button>
|
||||
</Tooltip>
|
||||
|
||||
{#if onInsertToNote && visibleResponseContent}
|
||||
<Tooltip content={$i18n.t('Insert into note')} placement="bottom">
|
||||
<button
|
||||
aria-label={$i18n.t('Insert into note')}
|
||||
class="{isLastMessage || ($settings?.highContrastMode ?? false)
|
||||
? 'visible'
|
||||
: 'hover-reveal'} rounded-lg px-2 py-1.5 text-xs text-gray-500 transition hover:bg-black/5 hover:text-black dark:hover:bg-white/5 dark:hover:text-white"
|
||||
on:click={() => {
|
||||
onInsertToNote?.(visibleResponseContent);
|
||||
}}
|
||||
>
|
||||
{$i18n.t('Insert')}
|
||||
</button>
|
||||
</Tooltip>
|
||||
{/if}
|
||||
|
||||
{#if !readOnly && ($user?.role === 'admin' || ($user?.permissions?.chat?.tts ?? true))}
|
||||
<Tooltip content={$i18n.t('Read Aloud')} placement="bottom">
|
||||
<button
|
||||
|
||||
@@ -1454,6 +1454,36 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b0/d8/00c6854ac1512bb9eaf13bd3f8f28222f7674947fc510a4ff7616f2efc80/google_resumable_media-2.10.0-py3-none-any.whl", hash = "sha256:88152884bee37b2bf36a0ab81ad8c7fd12212c9803dd981d77c1b35b02d34e7c", size = 81533, upload-time = "2026-06-03T16:13:12.51Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-re2"
|
||||
version = "1.1.20251105"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6b/60/805c654ba53d685513df955ee745f71920fe8e6a284faf0f9b9dc19b659c/google_re2-1.1.20251105.tar.gz", hash = "sha256:1db14a292ee8303b91e91e7c37e05ac17d3c467f29416c79ac70a78be3e65bda", size = 11676 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/4d/203a08dab1bdb5c83b46dd424c01a789ecb5a37dbc80f33d016bd116a9d7/google_re2-1.1.20251105-1-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:329efa209ea7baa44f0facf0402fa34e655dc97fdeb10d0b83fc06354f5575fd", size = 483717 },
|
||||
{ url = "https://files.pythonhosted.org/packages/78/88/466026b43ff5c7d740f5ede090992ec63b60d1810ab14fe35dfc00677e0a/google_re2-1.1.20251105-1-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:aa2ad5f6f48921ec137a7b7f1b1da903ddef8627a2dc30bc878a9a69d9925719", size = 515547 },
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/6a/c6c9fdb00c98990e4f7a6cd650e209d7b5d2754ca0404b72c69ac9909a69/google_re2-1.1.20251105-1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ac1cb2526cc88f050a0661fc7245ad009ee454bddc541b2e653f1d007585000d", size = 485396 },
|
||||
{ url = "https://files.pythonhosted.org/packages/a2/f6/529c44f607c47f96cfa29c1fe3a690fe75b2fdb48e9b0d6b54e5f0a75e59/google_re2-1.1.20251105-1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:50c7205182ad66c23c07abe8072f720ca2f7d595b61e28fd9b63623614f9afd6", size = 517150 },
|
||||
{ url = "https://files.pythonhosted.org/packages/df/d2/ccc07860e31ab81965c63f9ed4eb69ea0d3449a9b4e1610f71883694bbe8/google_re2-1.1.20251105-1-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:4cb5acee61e35772503b8b1db3c592a46b8e6a9bc0ab54d7d6233654ea2bf93d", size = 482807 },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/43/5fb20d16664457f61670bdd95f39039d43ee8b7732511c688e2f322a4317/google_re2-1.1.20251105-1-cp311-cp311-macosx_15_0_x86_64.whl", hash = "sha256:1617097d63620c2d46bdfc0e48f24f66cd341664fc75718636d234f67473fe7f", size = 508839 },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/f2/6e470338271e164dd3c5e508876f99aec3ed23bf419c7d54a5672fd5b05f/google_re2-1.1.20251105-1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18a5610b26742b90cb1d64ead2b16fe0e3bd7e67add03fd3779cd1b85e401661", size = 573718 },
|
||||
{ url = "https://files.pythonhosted.org/packages/91/21/4566fc344c21cf3c49082d13ddab785994b5e3b8b7fd4631242538f698a2/google_re2-1.1.20251105-1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03156291269f145eccddff63118f2df02d395792f51fc039f09955818943815a", size = 590749 },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/19/5981fb798bb8d08933b815b1fd9e55d179c380b9d8c21a49197b9b7c5967/google_re2-1.1.20251105-1-cp311-cp311-win32.whl", hash = "sha256:54f51762b51dc238eceddf49b56cc2b64594fe72d9328c1c39d615aa990e1f87", size = 434066 },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/e5/f83053a36cfc4762d843748e4f7a9c1141937dcf74cd6fc3f4598292dda3/google_re2-1.1.20251105-1-cp311-cp311-win_amd64.whl", hash = "sha256:f5f856ff5036a8f22b3bad57f376d4e3b97b59b64f311bdb1f83c8dabded2492", size = 491025 },
|
||||
{ url = "https://files.pythonhosted.org/packages/56/be/4315c3b38f42f9a2888fa76260545c98547502f1c35aa63a672d39011b2e/google_re2-1.1.20251105-1-cp311-cp311-win_arm64.whl", hash = "sha256:913864f97de4151eaa8bb7746ca230fd193656501e07fb658ce2cd46d4f6efcc", size = 642194 },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/20/73b487538e9107c2fd96aed737e3f3890dfce3e292622e4ffb2f9c810ee5/google_re2-1.1.20251105-1-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:b30f09b4d63249c72e65ccae4cbf6b331b48c22fc7cb439f1d85f347b9d07ceb", size = 485591 },
|
||||
{ url = "https://files.pythonhosted.org/packages/b9/9a/ca3a993bdb5dc6d5b2616b9657b2872a83d1827f8bd3ab50cd629eb751c7/google_re2-1.1.20251105-1-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:9a77892c524b8bdf3d47d7cad1cc2ac3a0108bdd65007ef4c02888fa46baf8ee", size = 518780 },
|
||||
{ url = "https://files.pythonhosted.org/packages/df/37/b2e367987371514253ec9e514637f457deaacb7acc1c900814f3a6421e0f/google_re2-1.1.20251105-1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a3ac51b28cbf25c100dfd8849212d878d7005d1d4a7e129a10789043c56b6021", size = 486966 },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/69/1db6742943c0ac254bfb7d8a37a5d3f73f016a65cfa1f84fe3a0451820f6/google_re2-1.1.20251105-1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:9f7158afc9825ac2654c6561aea94a1f7edb5b5b88e6e3639bb80bb817d102ac", size = 520225 },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/0a/0747c92dbebe2c09a26bd7386d372b5c5a9926236b4f3d69bb8f15db05cb/google_re2-1.1.20251105-1-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:5320da07dc3b7ac7f407514f42ac17d67e771ac7c7562d449571185e6fb601b2", size = 482943 },
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/14/6bfc6838bb6cb561824ac03deeab2bd11d5d9a93505f536c8fa2f6bd46c4/google_re2-1.1.20251105-1-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:5a4e5785bc30d52ce655d805b07ad2d8a4905429a5f690ae9c2f1caa76665709", size = 510384 },
|
||||
{ url = "https://files.pythonhosted.org/packages/8a/0a/6add090c917ee39f6f0be753037cafceb3bad904b424efc155fb38082635/google_re2-1.1.20251105-1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b7a3b90f747130310d4b3b8e19ebb845d0d97c1deb63b36f76c7242dacbd736", size = 572446 },
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/1c/8b1ccbeade96a21435d55b5185cd6d9b2ceab5a9af998a4d9099e0540759/google_re2-1.1.20251105-1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:809c5fa5d08279413b29c2e2c5c528e85cd94a0e0fd897db595a0c09eeee2782", size = 591348 },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/cf/7bdd7a1ae7828b613011da808eafec4da3132f43c3be6af5e0bd670ebe8b/google_re2-1.1.20251105-1-cp312-cp312-win32.whl", hash = "sha256:d8424e63a9ec0fe5bde03d97876b2431f8a746af33eb475fa1ae39144bd05b2a", size = 433787 },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/e9/5dd951c35acaabfe87c67228b9af2cdcd7779d9167edbe6b9094b8a8e529/google_re2-1.1.20251105-1-cp312-cp312-win_amd64.whl", hash = "sha256:062313c309f93dfeb6966372f4c446580e98879133ec155522eea8aaf568a5cd", size = 491726 },
|
||||
{ url = "https://files.pythonhosted.org/packages/60/8d/c1afd29fc2cb475fd4c634f3d3c8099c0efb662362c10b27a9eaf11c9357/google_re2-1.1.20251105-1-cp312-cp312-win_arm64.whl", hash = "sha256:558f144b26a9555ae4e9467cc3aa3299a8ce13217f328b21ae326ca0633be19b", size = 642673 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "googleapis-common-protos"
|
||||
version = "1.75.0"
|
||||
@@ -2756,6 +2786,7 @@ dependencies = [
|
||||
{ name = "faster-whisper" },
|
||||
{ name = "ftfy" },
|
||||
{ name = "google-cloud-storage" },
|
||||
{ name = "google-re2" },
|
||||
{ name = "googleapis-common-protos" },
|
||||
{ name = "hiredis" },
|
||||
{ name = "httpx", extra = ["brotli", "cli", "http2", "socks", "zstd"] },
|
||||
@@ -2882,6 +2913,7 @@ requires-dist = [
|
||||
{ name = "faster-whisper", specifier = "==1.2.1" },
|
||||
{ name = "ftfy", specifier = "==6.3.1" },
|
||||
{ name = "google-cloud-storage", specifier = "==3.9.0" },
|
||||
{ name = "google-re2", specifier = "==1.1.20251105" },
|
||||
{ name = "googleapis-common-protos", specifier = "==1.75.0" },
|
||||
{ name = "hiredis", specifier = "==3.4.0" },
|
||||
{ name = "httpx", extras = ["brotli", "cli", "http2", "socks", "zstd"], specifier = "==0.28.1" },
|
||||
|
||||
Reference in New Issue
Block a user