fix(websearch): add built-in provider request timeouts (#1874)

* Add timeouts for web search providers

* Fix WebSearch timeout body-stall test

* Address WebSearch timeout review feedback
This commit is contained in:
Bogdan
2026-07-07 21:54:11 +08:00
committed by GitHub
parent db01038d5c
commit 8599560b82
18 changed files with 758 additions and 100 deletions
@@ -69,6 +69,16 @@ export WEB_SEARCH_PROVIDER=tavily
export WEB_SEARCH_PROVIDER=auto
```
## Built-in Provider Timeout
Built-in adapter providers use a 15s request timeout so `auto` mode can fall through when a backend stalls. Override it with:
```bash
export WEB_SEARCH_TIMEOUT_SEC=30
```
Invalid, fractional, zero, negative, or very large values fall back to 15s. Custom API providers keep their separate `WEB_CUSTOM_TIMEOUT_SEC` setting because self-hosted endpoints may need different budgets.
## Provider Request & Response Formats
### Tavily
@@ -483,7 +493,7 @@ export WEB_JSON_PATH=response.payload.results
## Retry
Failed requests (network errors, 5xx) are retried once after 500ms. Client errors (4xx) are not retried. Custom requests have a default 120s timeout.
Failed custom-provider requests (network errors, 5xx) are retried once after 500ms. Client errors (4xx) are not retried. Custom requests have a default 120s timeout.
## Custom Provider Security Guardrails
+8 -8
View File
@@ -6,6 +6,7 @@
import type { SearchInput, SearchProvider } from './types.js'
import { applyDomainFilters, type ProviderOutput } from './types.js'
import { fetchJsonWithWebSearchTimeout } from './timeout.js'
export const bingProvider: SearchProvider = {
name: 'bing',
@@ -21,16 +22,15 @@ export const bingProvider: SearchProvider = {
url.searchParams.set('q', input.query)
url.searchParams.set('count', '15')
const res = await fetch(url.toString(), {
headers: { 'Ocp-Apim-Subscription-Key': process.env.BING_API_KEY! },
const data = await fetchJsonWithWebSearchTimeout(
url.toString(),
{
headers: { 'Ocp-Apim-Subscription-Key': process.env.BING_API_KEY! },
},
signal,
})
{ providerName: 'Bing' },
)
if (!res.ok) {
throw new Error(`Bing search error ${res.status}: ${await res.text().catch(() => '')}`)
}
const data = await res.json()
const hits = (data.webPages?.value ?? []).map((r: any) => ({
title: r.name ?? '',
url: r.url ?? '',
@@ -8,10 +8,18 @@ import { braveProvider } from './brave.ts'
const originalEnv = {
BRAVE_API_KEY: process.env.BRAVE_API_KEY,
WEB_SEARCH_TIMEOUT_SEC: process.env.WEB_SEARCH_TIMEOUT_SEC,
}
const originalFetch = globalThis.fetch
function stalledJsonResponse(status = 200): Response {
return new Response(new ReadableStream({ start() {} }), {
status,
headers: { 'Content-Type': 'application/json' },
})
}
beforeEach(async () => {
await acquireSharedMutationLock('WebSearchTool/providers/brave.test.ts')
})
@@ -104,6 +112,50 @@ describe('braveProvider search', () => {
await expect(braveProvider.search({ query: 'q' })).rejects.toThrow(/429/)
})
test('rejects when the provider-level timeout elapses', async () => {
process.env.WEB_SEARCH_TIMEOUT_SEC = '1'
let capturedSignal: AbortSignal | undefined
globalThis.fetch = (async (_input: any, init: any) => {
capturedSignal = init?.signal as AbortSignal | undefined
return new Promise<Response>(() => undefined)
}) as typeof fetch
await expect(braveProvider.search({ query: 'q' })).rejects.toThrow(
/Brave search timed out/,
)
expect(capturedSignal?.aborted).toBe(true)
})
test('rejects when the response body stalls after headers arrive', async () => {
process.env.WEB_SEARCH_TIMEOUT_SEC = '1'
globalThis.fetch = (async (_input: any, _init: any) => {
return stalledJsonResponse()
}) as typeof fetch
await expect(braveProvider.search({ query: 'q' })).rejects.toThrow(
/Brave search timed out/,
)
})
test('rejects when a non-2xx error body stalls after headers arrive', async () => {
process.env.WEB_SEARCH_TIMEOUT_SEC = '1'
let capturedSignal: AbortSignal | undefined
globalThis.fetch = (async (_input: any, init: any) => {
capturedSignal = init?.signal as AbortSignal | undefined
return stalledJsonResponse(500)
}) as typeof fetch
await expect(braveProvider.search({ query: 'q' })).rejects.toThrow(
/Brave search timed out/,
)
expect(capturedSignal?.aborted).toBe(true)
})
test('returns empty hits when web.results is missing', async () => {
globalThis.fetch = (async (_input: any, _init: any) =>
new Response(JSON.stringify({}), { status: 200 })) as typeof fetch
+10 -10
View File
@@ -9,6 +9,7 @@
import type { SearchInput, SearchProvider } from './types.js'
import { applyDomainFilters, safeHostname, type ProviderOutput } from './types.js'
import { fetchJsonWithWebSearchTimeout } from './timeout.js'
export const braveProvider: SearchProvider = {
name: 'brave',
@@ -24,19 +25,18 @@ export const braveProvider: SearchProvider = {
url.searchParams.set('q', input.query)
url.searchParams.set('count', '15')
const res = await fetch(url.toString(), {
headers: {
'X-Subscription-Token': process.env.BRAVE_API_KEY!,
Accept: 'application/json',
const data = await fetchJsonWithWebSearchTimeout(
url.toString(),
{
headers: {
'X-Subscription-Token': process.env.BRAVE_API_KEY!,
Accept: 'application/json',
},
},
signal,
})
{ providerName: 'Brave' },
)
if (!res.ok) {
throw new Error(`Brave search error ${res.status}: ${await res.text().catch(() => '')}`)
}
const data = await res.json()
const hits = (data.web?.results ?? []).map((r: any) => ({
title: r.title ?? '',
url: r.url ?? '',
@@ -1,4 +1,29 @@
import { describe, expect, test } from 'bun:test'
import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'
import {
acquireSharedMutationLock,
releaseSharedMutationLock,
} from '../../../test/sharedMutationLock.js'
const originalEnv = {
WEB_SEARCH_TIMEOUT_SEC: process.env.WEB_SEARCH_TIMEOUT_SEC,
}
beforeEach(async () => {
await acquireSharedMutationLock('WebSearchTool/providers/duckduckgo.test.ts')
})
afterEach(() => {
try {
if (originalEnv.WEB_SEARCH_TIMEOUT_SEC === undefined) {
delete process.env.WEB_SEARCH_TIMEOUT_SEC
} else {
process.env.WEB_SEARCH_TIMEOUT_SEC = originalEnv.WEB_SEARCH_TIMEOUT_SEC
}
mock.restore()
} finally {
releaseSharedMutationLock()
}
})
describe('DuckDuckGo SafeSearchType', () => {
test('SafeSearchType.STRICT === 0 (matches previous raw value)', async () => {
@@ -13,3 +38,66 @@ describe('DuckDuckGo SafeSearchType', () => {
expect(SafeSearchType.OFF).toBe(-2)
})
})
describe('duckduckgoProvider retry cancellation', () => {
test('provider-level timeout stops after one scrape attempt', async () => {
process.env.WEB_SEARCH_TIMEOUT_SEC = '1'
let calls = 0
let observedSignal: AbortSignal | undefined
mock.module('duck-duck-scrape', () => ({
SafeSearchType: {
STRICT: 0,
MODERATE: -1,
OFF: -2,
},
search: (
_query: string,
_options: unknown,
needleOptions?: { signal?: AbortSignal },
) => {
calls++
observedSignal = needleOptions?.signal
return new Promise(() => undefined)
},
}))
const { duckduckgoProvider } = await import('./duckduckgo.js')
await expect(
duckduckgoProvider.search({ query: 'provider timeout' }),
).rejects.toThrow(/DuckDuckGo search timed out/)
expect(calls).toBe(1)
expect(observedSignal).toBeInstanceOf(AbortSignal)
expect(observedSignal?.aborted).toBe(true)
})
test('caller abort during retry backoff stops without another attempt', async () => {
let calls = 0
mock.module('duck-duck-scrape', () => ({
SafeSearchType: {
STRICT: 0,
MODERATE: -1,
OFF: -2,
},
search: async () => {
calls++
throw new Error('timeout from DuckDuckGo')
},
}))
const { duckduckgoProvider } = await import('./duckduckgo.js')
const controller = new AbortController()
const abortTimer = setTimeout(() => controller.abort(), 10)
try {
await expect(
duckduckgoProvider.search({ query: 'retry abort' }, controller.signal),
).rejects.toMatchObject({ name: 'AbortError' })
} finally {
clearTimeout(abortTimer)
}
expect(calls).toBe(1)
})
})
@@ -1,5 +1,10 @@
import type { SearchInput, SearchProvider } from './types.js'
import { applyDomainFilters, type ProviderOutput } from './types.js'
import {
isWebSearchTimeoutError,
toAbortError,
withWebSearchTimeout,
} from './timeout.js'
// DuckDuckGo's HTML scraper aggressively blocks datacenter / repeat IPs with
// an "anomaly in the request" response. When that happens we surface an
@@ -23,6 +28,7 @@ function isAnomalyError(message: string): boolean {
function isRetryableDDGError(err: unknown): boolean {
if (!(err instanceof Error)) return false
if (isWebSearchTimeoutError(err)) return false
const msg = err.message.toLowerCase()
return (
msg.includes('anomaly') ||
@@ -35,8 +41,27 @@ function isRetryableDDGError(err: unknown): boolean {
)
}
function sleep(ms: number): Promise<void> {
return new Promise(r => setTimeout(r, ms))
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
if (signal?.aborted) {
return Promise.reject(toAbortError(signal.reason))
}
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
cleanup()
resolve()
}, ms)
const onAbort = () => {
cleanup()
reject(toAbortError(signal?.reason))
}
const cleanup = () => {
clearTimeout(timer)
signal?.removeEventListener('abort', onAbort)
}
signal?.addEventListener('abort', onAbort, { once: true })
})
}
export const duckduckgoProvider: SearchProvider = {
@@ -62,8 +87,16 @@ export const duckduckgoProvider: SearchProvider = {
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
if (signal?.aborted) throw new DOMException('Aborted', 'AbortError')
try {
// TODO: duck-duck-scrape doesn't accept AbortSignal — can't cancel in-flight searches
const response = await search(input.query, { safeSearch: SafeSearchType.STRICT })
const response = await withWebSearchTimeout(
combinedSignal =>
search(
input.query,
{ safeSearch: SafeSearchType.STRICT },
{ signal: combinedSignal } as Parameters<typeof search>[2],
),
signal,
{ providerName: 'DuckDuckGo' },
)
const hits = applyDomainFilters(
response.results.map(r => ({
@@ -91,7 +124,7 @@ export const duckduckgoProvider: SearchProvider = {
// Exponential backoff with jitter: 1s, 2s, 4s +/- 20%
const baseDelay = INITIAL_BACKOFF_MS * Math.pow(2, attempt)
const jitter = baseDelay * 0.2 * (Math.random() * 2 - 1)
await sleep(baseDelay + jitter)
await sleep(baseDelay + jitter, signal)
}
}
+12 -12
View File
@@ -21,6 +21,7 @@
import type { SearchInput, SearchProvider } from './types.js'
import { applyDomainFilters, safeHostname, type ProviderOutput } from './types.js'
import { fetchJsonWithWebSearchTimeout } from './timeout.js'
/** Join up to 3 highlight excerpts with an ellipsis separator. */
function describeFromHighlights(r: any): string | undefined {
@@ -52,21 +53,20 @@ export const exaProvider: SearchProvider = {
if (input.allowed_domains?.length) body.includeDomains = input.allowed_domains
if (input.blocked_domains?.length) body.excludeDomains = input.blocked_domains
const res = await fetch('https://api.exa.ai/search', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-api-key': process.env.EXA_API_KEY!,
const data = await fetchJsonWithWebSearchTimeout(
'https://api.exa.ai/search',
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-api-key': process.env.EXA_API_KEY!,
},
body: JSON.stringify(body),
},
body: JSON.stringify(body),
signal,
})
{ providerName: 'Exa' },
)
if (!res.ok) {
throw new Error(`Exa search error ${res.status}: ${await res.text().catch(() => '')}`)
}
const data = await res.json()
const hits = (data.results ?? []).map((r: any) => ({
title: r.title ?? '',
url: r.url ?? '',
+10 -5
View File
@@ -1,6 +1,7 @@
import type { SearchInput, SearchProvider } from './types.js'
import { applyDomainFilters, type ProviderOutput } from './types.js'
import { firecrawlSearch } from '../../firecrawl/client.js'
import { withWebSearchTimeout } from './timeout.js'
export const firecrawlProvider: SearchProvider = {
name: 'firecrawl',
@@ -19,12 +20,16 @@ export const firecrawlProvider: SearchProvider = {
query = `${query} ${exclusions}`
}
const data = await firecrawlSearch(query, {
apiKey: process.env.FIRECRAWL_API_KEY,
apiUrl: process.env.FIRECRAWL_API_URL,
limit: 15,
const data = await withWebSearchTimeout(
combinedSignal => firecrawlSearch(query, {
apiKey: process.env.FIRECRAWL_API_KEY,
apiUrl: process.env.FIRECRAWL_API_URL,
limit: 15,
signal: combinedSignal,
}),
signal,
})
{ providerName: 'Firecrawl' },
)
const hits = applyDomainFilters(
(data.web ?? []).map(r => ({
+129 -1
View File
@@ -1,4 +1,4 @@
import { describe, expect, test, beforeEach, afterEach } from 'bun:test'
import { describe, expect, test, beforeEach, afterEach, mock } from 'bun:test'
import {
acquireSharedMutationLock,
releaseSharedMutationLock,
@@ -8,9 +8,22 @@ import type { ProviderMode } from './index.js'
const savedWebSearchEnv = {
WEB_SEARCH_PROVIDER: process.env.WEB_SEARCH_PROVIDER,
WEB_SEARCH_TIMEOUT_SEC: process.env.WEB_SEARCH_TIMEOUT_SEC,
FIRECRAWL_API_KEY: process.env.FIRECRAWL_API_KEY,
FIRECRAWL_API_URL: process.env.FIRECRAWL_API_URL,
TAVILY_API_KEY: process.env.TAVILY_API_KEY,
EXA_API_KEY: process.env.EXA_API_KEY,
YOU_API_KEY: process.env.YOU_API_KEY,
JINA_API_KEY: process.env.JINA_API_KEY,
BRAVE_API_KEY: process.env.BRAVE_API_KEY,
BING_API_KEY: process.env.BING_API_KEY,
MOJEEK_API_KEY: process.env.MOJEEK_API_KEY,
LINKUP_API_KEY: process.env.LINKUP_API_KEY,
}
const originalFetch = globalThis.fetch
const originalConsoleError = console.error
function restoreWebSearchEnv() {
for (const [key, value] of Object.entries(savedWebSearchEnv)) {
if (value === undefined) {
@@ -28,11 +41,50 @@ beforeEach(async () => {
afterEach(() => {
try {
restoreWebSearchEnv()
globalThis.fetch = originalFetch
console.error = originalConsoleError
mock.restore()
} finally {
releaseSharedMutationLock()
}
})
function configureAutoModeWithOnlyBrave(): void {
process.env.WEB_SEARCH_PROVIDER = 'auto'
delete process.env.FIRECRAWL_API_KEY
delete process.env.FIRECRAWL_API_URL
delete process.env.TAVILY_API_KEY
delete process.env.EXA_API_KEY
delete process.env.YOU_API_KEY
delete process.env.JINA_API_KEY
process.env.BRAVE_API_KEY = 'brv-test-key'
delete process.env.BING_API_KEY
delete process.env.MOJEEK_API_KEY
delete process.env.LINKUP_API_KEY
}
function mockDuckDuckGoSearch(
search: () => Promise<{
results: Array<{ title: string; url: string; description?: string }>
}>,
): void {
mock.module('duck-duck-scrape', () => ({
SafeSearchType: {
STRICT: 0,
MODERATE: -1,
OFF: -2,
},
search,
}))
}
function stalledJsonResponse(): Response {
return new Response(new ReadableStream({ start() {} }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
})
}
// ---------------------------------------------------------------------------
// getProviderMode
// ---------------------------------------------------------------------------
@@ -145,6 +197,82 @@ describe('runSearch', () => {
else delete process.env.WEB_SEARCH_PROVIDER
}
})
test('auto mode falls through when a provider times out', async () => {
configureAutoModeWithOnlyBrave()
process.env.WEB_SEARCH_TIMEOUT_SEC = '1'
console.error = () => {}
globalThis.fetch = (async (_input: any, _init: any) =>
new Promise<Response>(() => undefined)) as typeof fetch
mockDuckDuckGoSearch(async () => ({
results: [
{
title: 'Fallback result',
url: 'https://example.com/fallback',
description: 'from ddg',
},
],
}))
const { runSearch } = await import('./index.js')
const output = await runSearch({ query: 'timeout fallback' })
expect(output.providerName).toBe('duckduckgo')
expect(output.hits).toHaveLength(1)
expect(output.hits[0].title).toBe('Fallback result')
})
test('auto mode falls through when a provider response body stalls', async () => {
configureAutoModeWithOnlyBrave()
process.env.WEB_SEARCH_TIMEOUT_SEC = '1'
console.error = () => {}
globalThis.fetch = (async (_input: any, _init?: any) =>
stalledJsonResponse()) as unknown as typeof fetch
mockDuckDuckGoSearch(async () => ({
results: [
{
title: 'Body fallback result',
url: 'https://example.com/body-fallback',
description: 'from ddg',
},
],
}))
const { runSearch } = await import('./index.js')
const output = await runSearch({ query: 'body timeout fallback' })
expect(output.providerName).toBe('duckduckgo')
expect(output.hits).toHaveLength(1)
expect(output.hits[0].title).toBe('Body fallback result')
})
test('auto mode does not fall through after caller abort', async () => {
configureAutoModeWithOnlyBrave()
process.env.WEB_SEARCH_TIMEOUT_SEC = '1'
let fetchCalls = 0
globalThis.fetch = (async (_input: any, _init: any) => {
fetchCalls++
return new Response(JSON.stringify({ web: { results: [] } }), { status: 200 })
}) as typeof fetch
let duckDuckGoCalls = 0
mockDuckDuckGoSearch(async () => {
duckDuckGoCalls++
return { results: [] }
})
const controller = new AbortController()
controller.abort()
const { runSearch } = await import('./index.js')
await expect(
runSearch({ query: 'user abort' }, controller.signal),
).rejects.toMatchObject({ name: 'AbortError' })
expect(fetchCalls).toBe(0)
expect(duckDuckGoCalls).toBe(0)
})
})
// ---------------------------------------------------------------------------
+10 -10
View File
@@ -6,6 +6,7 @@
import type { SearchInput, SearchProvider } from './types.js'
import { applyDomainFilters, safeHostname, type ProviderOutput } from './types.js'
import { fetchJsonWithWebSearchTimeout } from './timeout.js'
export const jinaProvider: SearchProvider = {
name: 'jina',
@@ -21,19 +22,18 @@ export const jinaProvider: SearchProvider = {
url.searchParams.set('q', input.query)
url.searchParams.set('count', '10')
const res = await fetch(url.toString(), {
headers: {
Authorization: `Bearer ${process.env.JINA_API_KEY}`,
Accept: 'application/json',
const data = await fetchJsonWithWebSearchTimeout(
url.toString(),
{
headers: {
Authorization: `Bearer ${process.env.JINA_API_KEY}`,
Accept: 'application/json',
},
},
signal,
})
{ providerName: 'Jina' },
)
if (!res.ok) {
throw new Error(`Jina search error ${res.status}: ${await res.text().catch(() => '')}`)
}
const data = await res.json()
const hits = (data.data ?? data.results ?? []).map((r: any) => ({
title: r.title ?? '',
url: r.url ?? '',
+16 -16
View File
@@ -6,6 +6,7 @@
import type { SearchInput, SearchProvider } from './types.js'
import { applyDomainFilters, safeHostname, type ProviderOutput } from './types.js'
import { fetchJsonWithWebSearchTimeout } from './timeout.js'
export const linkupProvider: SearchProvider = {
name: 'linkup',
@@ -17,25 +18,24 @@ export const linkupProvider: SearchProvider = {
async search(input: SearchInput, signal?: AbortSignal): Promise<ProviderOutput> {
const start = performance.now()
const res = await fetch('https://api.linkup.so/v1/search', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${process.env.LINKUP_API_KEY}`,
const data = await fetchJsonWithWebSearchTimeout(
'https://api.linkup.so/v1/search',
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${process.env.LINKUP_API_KEY}`,
},
body: JSON.stringify({
q: input.query,
search_type: 'standard',
depth: 'standard',
}),
},
body: JSON.stringify({
q: input.query,
search_type: 'standard',
depth: 'standard',
}),
signal,
})
{ providerName: 'Linkup' },
)
if (!res.ok) {
throw new Error(`Linkup search error ${res.status}: ${await res.text().catch(() => '')}`)
}
const data = await res.json()
const hits = (data.results ?? []).map((r: any) => ({
title: r.name ?? r.title ?? '',
url: r.url ?? '',
+9 -6
View File
@@ -6,6 +6,7 @@
import type { SearchInput, SearchProvider } from './types.js'
import { applyDomainFilters, safeHostname, type ProviderOutput } from './types.js'
import { fetchJsonWithWebSearchTimeout } from './timeout.js'
export const mojeekProvider: SearchProvider = {
name: 'mojeek',
@@ -29,13 +30,15 @@ export const mojeekProvider: SearchProvider = {
headers['Authorization'] = `Bearer ${process.env.MOJEEK_API_KEY}`
}
const res = await fetch(url.toString(), { headers, signal })
const data = await fetchJsonWithWebSearchTimeout(
url.toString(),
{
headers,
},
signal,
{ providerName: 'Mojeek' },
)
if (!res.ok) {
throw new Error(`Mojeek search error ${res.status}: ${await res.text().catch(() => '')}`)
}
const data = await res.json()
const rawResults = data?.response?.results ?? data?.results ?? []
const hits = rawResults.map((r: any) => ({
+16 -17
View File
@@ -6,6 +6,7 @@
import type { SearchInput, SearchProvider } from './types.js'
import { applyDomainFilters, safeHostname, type ProviderOutput } from './types.js'
import { fetchJsonWithWebSearchTimeout } from './timeout.js'
export const tavilyProvider: SearchProvider = {
name: 'tavily',
@@ -17,25 +18,23 @@ export const tavilyProvider: SearchProvider = {
async search(input: SearchInput, signal?: AbortSignal): Promise<ProviderOutput> {
const start = performance.now()
const res = await fetch('https://api.tavily.com/search', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${process.env.TAVILY_API_KEY}`,
const data = await fetchJsonWithWebSearchTimeout(
'https://api.tavily.com/search',
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${process.env.TAVILY_API_KEY}`,
},
body: JSON.stringify({
query: input.query,
max_results: 15,
include_answer: false,
}),
},
body: JSON.stringify({
query: input.query,
max_results: 15,
include_answer: false,
}),
signal,
})
if (!res.ok) {
throw new Error(`Tavily search error ${res.status}: ${await res.text().catch(() => '')}`)
}
const data = await res.json()
{ providerName: 'Tavily' },
)
const hits = (data.results ?? []).map((r: any) => ({
title: r.title ?? '',
@@ -0,0 +1,177 @@
import { describe, expect, test } from 'bun:test'
import {
DEFAULT_WEB_SEARCH_TIMEOUT_SECONDS,
getWebSearchTimeoutMs,
withWebSearchTimeout,
} from './timeout.js'
describe('getWebSearchTimeoutMs', () => {
test('returns the 15 second default', () => {
expect(getWebSearchTimeoutMs({} as NodeJS.ProcessEnv)).toBe(
DEFAULT_WEB_SEARCH_TIMEOUT_SECONDS * 1000,
)
})
test('uses WEB_SEARCH_TIMEOUT_SEC when it is a positive finite integer', () => {
expect(
getWebSearchTimeoutMs({
WEB_SEARCH_TIMEOUT_SEC: '2',
} as NodeJS.ProcessEnv),
).toBe(2000)
})
test('falls back to the default for invalid or absurd values', () => {
for (const value of ['', '0', '-1', '0.5', '1.5', '2.0', 'nope', 'Infinity', '999999']) {
expect(
getWebSearchTimeoutMs({
WEB_SEARCH_TIMEOUT_SEC: value,
} as NodeJS.ProcessEnv),
).toBe(DEFAULT_WEB_SEARCH_TIMEOUT_SECONDS * 1000)
}
})
test('accepts the configured maximum timeout and rejects values above it', () => {
expect(
getWebSearchTimeoutMs({
WEB_SEARCH_TIMEOUT_SEC: '300',
} as NodeJS.ProcessEnv),
).toBe(300_000)
expect(
getWebSearchTimeoutMs({
WEB_SEARCH_TIMEOUT_SEC: '301',
} as NodeJS.ProcessEnv),
).toBe(DEFAULT_WEB_SEARCH_TIMEOUT_SECONDS * 1000)
})
})
describe('withWebSearchTimeout', () => {
test('rejects a never-resolving operation on timeout', async () => {
await expect(
withWebSearchTimeout(
() => new Promise(() => undefined),
undefined,
{ providerName: 'TestSearch', timeoutMs: 5 },
),
).rejects.toThrow(/TestSearch search timed out/)
})
test('timeout errors carry a stable marker', async () => {
await expect(
withWebSearchTimeout(
() => new Promise(() => undefined),
undefined,
{ providerName: 'TestSearch', timeoutMs: 5 },
),
).rejects.toMatchObject({
name: 'WebSearchTimeoutError',
code: 'WEB_SEARCH_TIMEOUT',
timeoutMs: 5,
})
})
test('keeps caller aborts as AbortError and does not start work', async () => {
const controller = new AbortController()
controller.abort()
let started = false
await expect(
withWebSearchTimeout(
async () => {
started = true
return 'unexpected'
},
controller.signal,
{ providerName: 'TestSearch', timeoutMs: 5 },
),
).rejects.toMatchObject({ name: 'AbortError' })
expect(started).toBe(false)
})
test('races non-signal-aware DuckDuckGo-style work at the timeout boundary', async () => {
let observedSignal: AbortSignal | undefined
await expect(
withWebSearchTimeout(
signal => {
observedSignal = signal
return new Promise(() => undefined)
},
undefined,
{ providerName: 'DuckDuckGo', timeoutMs: 5 },
),
).rejects.toThrow(/DuckDuckGo search timed out/)
expect(observedSignal?.aborted).toBe(true)
})
test('removes caller abort listeners after successful operations', async () => {
const controller = new AbortController()
const addEventListener = controller.signal.addEventListener.bind(controller.signal)
const removeEventListener = controller.signal.removeEventListener.bind(controller.signal)
let abortListenersAdded = 0
let abortListenersRemoved = 0
controller.signal.addEventListener = ((type, listener, options) => {
if (type === 'abort') abortListenersAdded++
return addEventListener(type, listener, options)
}) as typeof controller.signal.addEventListener
controller.signal.removeEventListener = ((type, listener, options) => {
if (type === 'abort') abortListenersRemoved++
return removeEventListener(type, listener, options)
}) as typeof controller.signal.removeEventListener
try {
await expect(
withWebSearchTimeout(
async () => 'ok',
controller.signal,
{ providerName: 'TestSearch', timeoutMs: 100 },
),
).resolves.toBe('ok')
} finally {
controller.signal.addEventListener = addEventListener
controller.signal.removeEventListener = removeEventListener
}
expect(abortListenersAdded).toBeGreaterThan(0)
expect(abortListenersRemoved).toBe(abortListenersAdded)
})
test('removes caller abort listeners after timed-out operations', async () => {
const controller = new AbortController()
const addEventListener = controller.signal.addEventListener.bind(controller.signal)
const removeEventListener = controller.signal.removeEventListener.bind(controller.signal)
let abortListenersAdded = 0
let abortListenersRemoved = 0
controller.signal.addEventListener = ((type, listener, options) => {
if (type === 'abort') abortListenersAdded++
return addEventListener(type, listener, options)
}) as typeof controller.signal.addEventListener
controller.signal.removeEventListener = ((type, listener, options) => {
if (type === 'abort') abortListenersRemoved++
return removeEventListener(type, listener, options)
}) as typeof controller.signal.removeEventListener
try {
await expect(
withWebSearchTimeout(
() => new Promise(() => undefined),
controller.signal,
{ providerName: 'TestSearch', timeoutMs: 5 },
),
).rejects.toMatchObject({ code: 'WEB_SEARCH_TIMEOUT' })
} finally {
controller.signal.addEventListener = addEventListener
controller.signal.removeEventListener = removeEventListener
}
expect(abortListenersAdded).toBeGreaterThan(0)
expect(abortListenersRemoved).toBe(abortListenersAdded)
})
})
@@ -0,0 +1,157 @@
import { createCombinedAbortSignal } from '../../../utils/combinedAbortSignal.js'
export const DEFAULT_WEB_SEARCH_TIMEOUT_SECONDS = 15
const MAX_WEB_SEARCH_TIMEOUT_SECONDS = 300
const WEB_SEARCH_TIMEOUT_CODE = 'WEB_SEARCH_TIMEOUT'
interface WebSearchTimeoutOptions {
providerName?: string
timeoutMs?: number
}
export function getWebSearchTimeoutMs(
env: NodeJS.ProcessEnv = process.env,
): number {
const raw = env.WEB_SEARCH_TIMEOUT_SEC
const trimmed = raw?.trim()
if (trimmed === undefined || trimmed === '') {
return DEFAULT_WEB_SEARCH_TIMEOUT_SECONDS * 1000
}
if (!/^\d+$/.test(trimmed)) {
return DEFAULT_WEB_SEARCH_TIMEOUT_SECONDS * 1000
}
const seconds = Number(trimmed)
if (
!Number.isFinite(seconds) ||
!Number.isSafeInteger(seconds) ||
seconds <= 0 ||
seconds > MAX_WEB_SEARCH_TIMEOUT_SECONDS
) {
return DEFAULT_WEB_SEARCH_TIMEOUT_SECONDS * 1000
}
return seconds * 1000
}
export class WebSearchTimeoutError extends Error {
readonly code = WEB_SEARCH_TIMEOUT_CODE
readonly timeoutMs: number
constructor(providerName: string, timeoutMs: number) {
super(
`${providerName} search timed out after ${formatTimeoutSeconds(timeoutMs)}s`,
)
this.name = 'WebSearchTimeoutError'
this.timeoutMs = timeoutMs
}
}
export function isWebSearchTimeoutError(
err: unknown,
): err is WebSearchTimeoutError {
return (
err instanceof WebSearchTimeoutError ||
(err instanceof Error &&
(err as { code?: unknown }).code === WEB_SEARCH_TIMEOUT_CODE)
)
}
export function toAbortError(reason?: unknown): Error {
if (reason instanceof Error) return reason
return new DOMException('Aborted', 'AbortError')
}
function waitForAbort(signal: AbortSignal): {
promise: Promise<never>
cleanup: () => void
} {
let onAbort: (() => void) | undefined
const promise = new Promise<never>((_, reject) => {
if (signal.aborted) {
reject(toAbortError(signal.reason))
return
}
onAbort = () => reject(toAbortError(signal.reason))
signal.addEventListener('abort', onAbort, { once: true })
})
return {
promise,
cleanup: () => {
if (onAbort) signal.removeEventListener('abort', onAbort)
onAbort = undefined
},
}
}
function formatTimeoutSeconds(timeoutMs: number): string {
return String(timeoutMs / 1000)
}
export async function withWebSearchTimeout<T>(
operation: (signal: AbortSignal) => Promise<T>,
signal?: AbortSignal,
options: WebSearchTimeoutOptions = {},
): Promise<T> {
const timeoutMs = options.timeoutMs ?? getWebSearchTimeoutMs()
const providerName = options.providerName ?? 'Web search provider'
const { signal: combined, cleanup } = createCombinedAbortSignal(signal, {
timeoutMs,
})
try {
if (combined.aborted) {
throw toAbortError(combined.reason)
}
const abortWait = waitForAbort(combined)
try {
return await Promise.race([
operation(combined),
abortWait.promise,
])
} finally {
abortWait.cleanup()
}
} catch (err) {
if (signal?.aborted) {
throw toAbortError(signal.reason ?? err)
}
if (combined.aborted) {
throw new WebSearchTimeoutError(providerName, timeoutMs)
}
throw err
} finally {
cleanup()
}
}
export async function fetchJsonWithWebSearchTimeout(
input: Parameters<typeof fetch>[0],
init: Parameters<typeof fetch>[1],
signal: AbortSignal | undefined,
options: WebSearchTimeoutOptions = {},
): Promise<any> {
const providerName = options.providerName ?? 'Web search provider'
return withWebSearchTimeout(
async combinedSignal => {
const res = await fetch(input, {
...(init ?? {}),
signal: combinedSignal,
})
if (!res.ok) {
throw new Error(
`${providerName} search error ${res.status}: ${await res.text().catch(() => '')}`,
)
}
return await res.json()
},
signal,
options,
)
}
+8 -8
View File
@@ -6,6 +6,7 @@
import type { SearchInput, SearchProvider } from './types.js'
import { applyDomainFilters, safeHostname, type ProviderOutput } from './types.js'
import { fetchJsonWithWebSearchTimeout } from './timeout.js'
export const youProvider: SearchProvider = {
name: 'you',
@@ -21,16 +22,15 @@ export const youProvider: SearchProvider = {
url.searchParams.set('query', input.query)
url.searchParams.set('num_web_results', '10')
const res = await fetch(url.toString(), {
headers: { 'X-API-Key': process.env.YOU_API_KEY! },
const data = await fetchJsonWithWebSearchTimeout(
url.toString(),
{
headers: { 'X-API-Key': process.env.YOU_API_KEY! },
},
signal,
})
{ providerName: 'You.com' },
)
if (!res.ok) {
throw new Error(`You.com search error ${res.status}: ${await res.text().catch(() => '')}`)
}
const data = await res.json()
const webResults = data?.results?.web ?? data?.results ?? []
const hits = webResults.map((r: any) => {
+5
View File
@@ -15,6 +15,7 @@ import {
const TEST_ENV_KEYS = [
'NODE_OPTIONS',
'AZURE_OPENAI_API_VERSION',
'CLAUDE_CODE_USE_OPENAI',
'CODEX_AUTH_JSON_PATH',
'CODEX_HOME',
'OPENAI_API_KEYS',
@@ -38,6 +39,7 @@ const TEST_ENV_KEYS = [
'WEB_QUERY_PARAM',
'WEB_SEARCH_API',
'WEB_SEARCH_PROVIDER',
'WEB_SEARCH_TIMEOUT_SEC',
'WEB_URL_TEMPLATE',
]
@@ -298,6 +300,7 @@ describe('loadEnvFile', () => {
'WEB_AUTH_SCHEME=',
'WEB_HEADERS=Accept: application/json; X-Tenant: acme',
'WEB_JSON_PATH=response.payload.results',
'WEB_SEARCH_TIMEOUT_SEC=30',
'WEB_CUSTOM_TIMEOUT_SEC=15',
'WEB_CUSTOM_MAX_BODY_KB=300',
'WEB_CUSTOM_ALLOW_ARBITRARY_HEADERS=true',
@@ -323,6 +326,7 @@ describe('loadEnvFile', () => {
WEB_AUTH_SCHEME: '',
WEB_HEADERS: 'Accept: application/json; X-Tenant: acme',
WEB_JSON_PATH: 'response.payload.results',
WEB_SEARCH_TIMEOUT_SEC: '30',
WEB_CUSTOM_TIMEOUT_SEC: '15',
WEB_CUSTOM_MAX_BODY_KB: '300',
WEB_CUSTOM_ALLOW_ARBITRARY_HEADERS: 'true',
@@ -332,6 +336,7 @@ describe('loadEnvFile', () => {
CODEX_HOME: '/tmp/codex',
})
expect(process.env.WEB_SEARCH_API).toBe('https://search.example.com/search')
expect(process.env.WEB_SEARCH_TIMEOUT_SEC).toBe('30')
expect(process.env.CODEX_AUTH_JSON_PATH).toBe('/tmp/codex-auth.json')
})
+1
View File
@@ -118,6 +118,7 @@ const ALLOWED_ENV_FILE_KEYS = new Set([
'WEB_QUERY_PARAM',
'WEB_SEARCH_API',
'WEB_SEARCH_PROVIDER',
'WEB_SEARCH_TIMEOUT_SEC',
'WEB_URL_TEMPLATE',
'XAI_API_KEY',
'XAI_CREDENTIAL_SOURCE',