mirror of
https://github.com/Gitlawb/openclaude.git
synced 2026-11-06 12:50:54 -04:00
feat: add AnySearch as an explicit web search backend (#2261)
* feat: add AnySearch web search backend * fix(anysearch): preserve non-json error details --------- Co-authored-by: liuyang <liuy.web@lingyu.work>
This commit is contained in:
@@ -408,9 +408,13 @@ See [Agent Routing and Step Limits](docs/agent-routing.md) for the full guide.
|
||||
/search remove-key # remove a saved Exa key
|
||||
/search status # show which backend handles searches
|
||||
/search test # run a test search
|
||||
/search anysearch # select AnySearch (works without a key)
|
||||
/search key anysearch # optionally add an AnySearch API key in the hidden input
|
||||
```
|
||||
|
||||
Any other backend you configure with a key (Tavily, Brave, Firecrawl, …) runs before the Exa free tier, and DuckDuckGo is the last-resort fallback. When the active provider is Ollama, OpenClaude uses the signed-in local Ollama Web Search endpoint ahead of the free tier. Set `OLLAMA_API_KEY` to enable the hosted Ollama endpoint as a fallback. Set `EXA_FREE_TIER=0` if searches should never go to Exa without a key. See [Web Search Providers](src/tools/WebSearchTool/README_SEARCH_PROVIDERS.md) for every backend and option.
|
||||
Other backends in the auto chain that you configure with a key (Tavily, Brave, Firecrawl, …) run before the Exa free tier, and DuckDuckGo is the last-resort fallback. When the active provider is Ollama, OpenClaude uses the signed-in local Ollama Web Search endpoint ahead of the free tier. Set `OLLAMA_API_KEY` to enable the hosted Ollama endpoint as a fallback. Set `EXA_FREE_TIER=0` if searches should never go to Exa without a key. See [Web Search Providers](src/tools/WebSearchTool/README_SEARCH_PROVIDERS.md) for every backend and option.
|
||||
|
||||
AnySearch is available as an explicit backend with `/search anysearch` or `WEB_SEARCH_PROVIDER=anysearch`. `ANYSEARCH_API_KEY` is optional; anonymous access has lower service limits. Set `ANYSEARCH_MAX_RESULTS` to an integer from 1 to 10 (default 10) to control the requested result count. AnySearch does not change the `auto` search order.
|
||||
|
||||
> **Note:** DuckDuckGo fallback works by scraping search results and may be rate-limited, blocked, or subject to DuckDuckGo's Terms of Service.
|
||||
|
||||
|
||||
@@ -557,7 +557,9 @@ search (`/search test`). `/search` saves its choices to the `env` block of
|
||||
|
||||
| Variable | Required | Description |
|
||||
|----------|----------|-------------|
|
||||
| `WEB_SEARCH_PROVIDER` | No | `auto` (default): keyed Exa first, then any other backend you configured with a key, then the Exa free tier, then DuckDuckGo. Anthropic first-party, Vertex, Foundry, and Codex keep their built-in search in `auto`. Set one backend (`exa`, `tavily`, `brave`, `firecrawl`, `you`, `jina`, `bing`, `mojeek`, `linkup`, `ollama`, `ddg`, `custom`, or `native`) to use only that backend and fail instead of falling back. |
|
||||
| `WEB_SEARCH_PROVIDER` | No | `auto` (default): keyed Exa first, then other configured backends in the auto chain, then the Exa free tier, then DuckDuckGo. Anthropic first-party, Vertex, Foundry, and Codex keep their built-in search in `auto`. Set one backend (`exa`, `anysearch`, `tavily`, `brave`, `firecrawl`, `you`, `jina`, `bing`, `mojeek`, `linkup`, `ollama`, `ddg`, `custom`, or `native`) to use only that backend and fail instead of falling back. AnySearch is explicit-only and does not join `auto`. |
|
||||
| `ANYSEARCH_API_KEY` | No | Optional AnySearch API key. Without one, `/search anysearch` sends anonymous requests without an Authorization header. Add or remove a saved key with `/search key anysearch` or `/search remove-key anysearch`. |
|
||||
| `ANYSEARCH_MAX_RESULTS` | No | Requested AnySearch results, integer 1–10 (default `10`; invalid values use the default). |
|
||||
| `EXA_API_KEY` | No | Exa API key for higher limits than the free tier. Free keys: https://dashboard.exa.ai/api-keys |
|
||||
| `EXA_SEARCH_TYPE` | No | Exa search type with a key: `auto` (default), `instant`, `fast`, `deep-lite`, `deep`, or `deep-reasoning`. The deep types can need a higher `WEB_SEARCH_TIMEOUT_SEC`. |
|
||||
| `EXA_NUM_RESULTS` | No | Results per Exa search, 1–50 (default `15`). |
|
||||
|
||||
@@ -31,8 +31,9 @@ const memoryStore: SearchEnvStore = {
|
||||
const SYNC_START = '\x1B[?2026h'
|
||||
const SYNC_END = '\x1B[?2026l'
|
||||
|
||||
const ENV_KEYS = ['WEB_SEARCH_PROVIDER', 'EXA_API_KEY', 'TAVILY_API_KEY'] as const
|
||||
const ENV_KEYS = ['WEB_SEARCH_PROVIDER', 'EXA_API_KEY', 'TAVILY_API_KEY', 'ANYSEARCH_API_KEY'] as const
|
||||
const savedEnv = Object.fromEntries(ENV_KEYS.map(k => [k, process.env[k]]))
|
||||
const originalFetch = globalThis.fetch
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock('commands/search/search.test.tsx')
|
||||
@@ -44,6 +45,7 @@ beforeEach(async () => {
|
||||
afterEach(() => {
|
||||
try {
|
||||
setSearchEnvStoreForTests(undefined)
|
||||
globalThis.fetch = originalFetch
|
||||
for (const [key, value] of Object.entries(savedEnv)) {
|
||||
if (value === undefined) delete process.env[key]
|
||||
else process.env[key] = value
|
||||
@@ -184,9 +186,18 @@ describe('/search command', () => {
|
||||
|
||||
expect(frame).toContain('Auto (recommended)')
|
||||
expect(frame).toContain('Exa')
|
||||
expect(frame).toContain('AnySearch')
|
||||
expect(frame).toContain('needs TAVILY_API_KEY')
|
||||
})
|
||||
|
||||
test('direct selection enables AnySearch without opening a key dialog', async () => {
|
||||
const { text, mounted } = await runSearch('anysearch')
|
||||
|
||||
expect(mounted).toBeUndefined()
|
||||
expect(text).toBe('Web search set to AnySearch.')
|
||||
expect(savedEnvBlock).toEqual({ WEB_SEARCH_PROVIDER: 'anysearch' })
|
||||
})
|
||||
|
||||
test('typing a key saves the backend and key without ever showing the key', async () => {
|
||||
const { mounted } = await runSearch('tavily')
|
||||
await mounted!.waitFor(f => f.includes('Tavily API key'))
|
||||
@@ -206,6 +217,38 @@ describe('/search command', () => {
|
||||
expect(process.env.TAVILY_API_KEY).toBe('tvly-secret-123')
|
||||
})
|
||||
|
||||
test('an AnySearch key can be saved through hidden input and removed', async () => {
|
||||
await runSearch('anysearch')
|
||||
const { mounted } = await runSearch('key anysearch')
|
||||
await mounted!.waitFor(f => f.includes('AnySearch API key'))
|
||||
|
||||
mounted!.stdin.write('private-any-key')
|
||||
const masked = await mounted!.waitFor(f => f.includes('***************'))
|
||||
mounted!.stdin.write('\r')
|
||||
const message = await waitUntil(mounted!.done, text => text !== undefined)
|
||||
mounted!.unmount()
|
||||
|
||||
expect(masked).not.toContain('private-any-key')
|
||||
expect(message).toBe('Saved ANYSEARCH_API_KEY. AnySearch now uses your key.')
|
||||
expect(savedEnvBlock).toEqual({
|
||||
WEB_SEARCH_PROVIDER: 'anysearch',
|
||||
ANYSEARCH_API_KEY: 'private-any-key',
|
||||
})
|
||||
|
||||
const removed = await runSearch('remove-key anysearch')
|
||||
expect(removed.text).toBe('Removed the saved ANYSEARCH_API_KEY.')
|
||||
expect(process.env.ANYSEARCH_API_KEY).toBeUndefined()
|
||||
|
||||
let authorizationPresent = true
|
||||
globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
authorizationPresent = Object.hasOwn(init?.headers ?? {}, 'Authorization')
|
||||
return new Response(JSON.stringify({ code: 0, data: { results: [] } }), { status: 200 })
|
||||
}) as typeof fetch
|
||||
const anonymousTest = await runSearch('test anonymous after removing key')
|
||||
expect(anonymousTest.text).toContain('Search OK via AnySearch: 0 results')
|
||||
expect(authorizationPresent).toBe(false)
|
||||
})
|
||||
|
||||
test('Esc in the key dialog cancels without saving', async () => {
|
||||
const { mounted } = await runSearch('key exa')
|
||||
await mounted!.waitFor(f => f.includes('Exa API key'))
|
||||
|
||||
@@ -33,6 +33,8 @@ const SEARCH_ENV_KEYS = [
|
||||
'EXA_API_KEY',
|
||||
'EXA_FREE_TIER',
|
||||
'TAVILY_API_KEY',
|
||||
'ANYSEARCH_API_KEY',
|
||||
'ANYSEARCH_MAX_RESULTS',
|
||||
'FIRECRAWL_API_KEY',
|
||||
'FIRECRAWL_API_URL',
|
||||
'YOU_API_KEY',
|
||||
@@ -50,6 +52,7 @@ const SEARCH_ENV_KEYS = [
|
||||
] as const
|
||||
|
||||
const savedEnv = Object.fromEntries(SEARCH_ENV_KEYS.map(k => [k, process.env[k]]))
|
||||
const originalFetch = globalThis.fetch
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock('commands/search/searchSettings.test.ts')
|
||||
@@ -58,6 +61,7 @@ beforeEach(async () => {
|
||||
|
||||
afterEach(() => {
|
||||
try {
|
||||
globalThis.fetch = originalFetch
|
||||
for (const [key, value] of Object.entries(savedEnv)) {
|
||||
if (value === undefined) delete process.env[key]
|
||||
else process.env[key] = value
|
||||
@@ -124,6 +128,11 @@ describe('parseSearchArgs', () => {
|
||||
})
|
||||
expect(parseSearchArgs('duckduckgo')).toMatchObject({ kind: 'set', option: { mode: 'ddg' } })
|
||||
expect(parseSearchArgs('builtin')).toMatchObject({ kind: 'set', option: { mode: 'native' } })
|
||||
expect(parseSearchArgs('anysearch')).toMatchObject({
|
||||
kind: 'set',
|
||||
option: { mode: 'anysearch' },
|
||||
typedInline: false,
|
||||
})
|
||||
})
|
||||
|
||||
test('extra tokens after a backend are flagged as an inline key', () => {
|
||||
@@ -194,6 +203,7 @@ describe('needsApiKey', () => {
|
||||
expect(needsApiKey(backend('exa'))).toBe(false)
|
||||
expect(needsApiKey(backend('ddg'))).toBe(false)
|
||||
expect(needsApiKey(backend('auto'))).toBe(false)
|
||||
expect(needsApiKey(backend('anysearch'))).toBe(false)
|
||||
})
|
||||
|
||||
test('Exa needs a key when the free tier is turned off', () => {
|
||||
@@ -235,6 +245,11 @@ describe('describeSearchStatus', () => {
|
||||
)
|
||||
})
|
||||
|
||||
test('an explicit AnySearch selection reports the active backend without a key', () => {
|
||||
process.env.WEB_SEARCH_PROVIDER = 'anysearch'
|
||||
expect(describeSearchStatus(false)).toContain('Searches go to: AnySearch')
|
||||
})
|
||||
|
||||
test('native mode without built-in search reports searches as unavailable', () => {
|
||||
process.env.WEB_SEARCH_PROVIDER = 'native'
|
||||
expect(describeSearchStatus(false)).toContain('searches are unavailable')
|
||||
@@ -242,6 +257,16 @@ describe('describeSearchStatus', () => {
|
||||
})
|
||||
|
||||
describe('applySearchSelection', () => {
|
||||
test('saves AnySearch selection without requiring an API key', () => {
|
||||
const store = recordingStore()
|
||||
const message = applySearchSelection(backend('anysearch'), undefined, false, store)
|
||||
|
||||
expect(store.saves).toEqual([
|
||||
{ set: { WEB_SEARCH_PROVIDER: 'anysearch' }, unset: [] },
|
||||
])
|
||||
expect(message).toBe('Web search set to AnySearch.')
|
||||
})
|
||||
|
||||
test('saves the backend and its key, and applies them to this session', () => {
|
||||
const store = recordingStore()
|
||||
const message = applySearchSelection(backend('tavily'), ' tvly-key ', false, store)
|
||||
@@ -456,6 +481,24 @@ describe('runSearchTest', () => {
|
||||
durationSeconds: 1.234,
|
||||
}
|
||||
|
||||
test('uses the selected AnySearch backend through the normal test entry', async () => {
|
||||
process.env.WEB_SEARCH_PROVIDER = 'anysearch'
|
||||
let requestedUrl = ''
|
||||
globalThis.fetch = (async (input: RequestInfo | URL, _init?: RequestInit) => {
|
||||
requestedUrl = String(input)
|
||||
return new Response(JSON.stringify({
|
||||
code: 0,
|
||||
data: { results: [{ title: 'Any result', url: 'https://example.com/result' }] },
|
||||
}), { status: 200 })
|
||||
}) as typeof fetch
|
||||
|
||||
const message = await runSearchTest('any query', false)
|
||||
|
||||
expect(requestedUrl).toBe('https://api.anysearch.com/v1/search')
|
||||
expect(message).toContain('Search OK via AnySearch: 1 results')
|
||||
expect(message).toContain('Top result: Any result — https://example.com/result')
|
||||
})
|
||||
|
||||
test('reports the backend, count, timing, and top result', async () => {
|
||||
let query = ''
|
||||
const message = await runSearchTest('node release', false, async input => {
|
||||
|
||||
@@ -45,7 +45,7 @@ export const SEARCH_BACKENDS: readonly SearchBackendOption[] = [
|
||||
{
|
||||
mode: 'auto',
|
||||
label: 'Auto (recommended)',
|
||||
description: 'Exa first, then any backend you configured, then free fallbacks',
|
||||
description: 'Exa first, then configured auto backends, then free fallbacks',
|
||||
},
|
||||
{
|
||||
mode: 'exa',
|
||||
@@ -54,6 +54,13 @@ export const SEARCH_BACKENDS: readonly SearchBackendOption[] = [
|
||||
keyOptional: true,
|
||||
description: 'Exa only — free tier without a key, higher limits with one',
|
||||
},
|
||||
{
|
||||
mode: 'anysearch',
|
||||
label: 'AnySearch',
|
||||
keyEnv: 'ANYSEARCH_API_KEY',
|
||||
keyOptional: true,
|
||||
description: 'AnySearch only — anonymous access without a key, higher limits with one',
|
||||
},
|
||||
{ mode: 'tavily', label: 'Tavily', keyEnv: 'TAVILY_API_KEY', description: 'Tavily only' },
|
||||
{
|
||||
mode: 'brave',
|
||||
@@ -95,6 +102,7 @@ const BACKEND_ALIASES: Record<string, ProviderMode> = {
|
||||
const BACKEND_DISPLAY_NAMES: Record<string, string> = {
|
||||
exa: 'Exa (API key)',
|
||||
'exa-free': 'Exa free tier',
|
||||
anysearch: 'AnySearch',
|
||||
ollama: 'Ollama',
|
||||
firecrawl: 'Firecrawl',
|
||||
tavily: 'Tavily',
|
||||
|
||||
@@ -46,6 +46,7 @@ instead.
|
||||
| Tavily | `TAVILY_API_KEY` | `Authorization: Bearer` | POST |
|
||||
| Exa (default) | `EXA_API_KEY` | `x-api-key` | POST |
|
||||
| Exa free tier | *(none — zero-config default)* | — | POST |
|
||||
| AnySearch (explicit only) | `ANYSEARCH_API_KEY` (optional) | `Authorization: Bearer` when set | POST |
|
||||
| You.com | `YOU_API_KEY` | `X-API-Key` | GET |
|
||||
| Jina | `JINA_API_KEY` | `Authorization: Bearer` | GET |
|
||||
| Bing | `BING_API_KEY` | `Ocp-Apim-Subscription-Key` | GET |
|
||||
@@ -83,6 +84,7 @@ export WEB_SEARCH_API=https://search.example.com/search
|
||||
| `ollama` | Local signed-in Ollama, then hosted Ollama when `OLLAMA_API_KEY` is set; throws if both fail |
|
||||
| `tavily` | Tavily only — throws on failure |
|
||||
| `exa` | Exa only — keyed API when `EXA_API_KEY` is set, otherwise the free tier; throws on failure |
|
||||
| `anysearch` | AnySearch only — anonymous without a key, Bearer authentication with one; throws on failure |
|
||||
| `brave` | Brave only — throws on failure |
|
||||
| `custom` | Custom API only — throws on failure. **Not in the auto chain** — must be explicitly selected |
|
||||
| `firecrawl` | Firecrawl only — throws on failure |
|
||||
@@ -101,6 +103,8 @@ there too.
|
||||
|
||||
> **Note:** The `custom` provider is excluded from the `auto` chain. It is only used when `WEB_SEARCH_PROVIDER=custom` is explicitly set. This prevents the generic outbound provider from silently becoming the default backend.
|
||||
|
||||
AnySearch is also explicit-only: selecting it does not change the existing `auto` chain.
|
||||
|
||||
```bash
|
||||
# Fail loudly if Tavily is down (don't silently switch backends)
|
||||
export WEB_SEARCH_PROVIDER=tavily
|
||||
@@ -121,6 +125,16 @@ Invalid, fractional, zero, negative, or very large values fall back to 15s. Cust
|
||||
|
||||
## Provider Request & Response Formats
|
||||
|
||||
### AnySearch
|
||||
|
||||
Run `/search anysearch` to use its general web search without a key, or `/search key anysearch` to add an optional key through hidden input. Anonymous access has lower service limits. `/search remove-key anysearch` removes a key saved by `/search`; keys from the shell or `settings.json` remain controlled by those sources.
|
||||
|
||||
`ANYSEARCH_MAX_RESULTS` accepts an integer from 1 to 10 (default 10). Invalid values use 10. General search sends `query` and `max_results` to `POST https://api.anysearch.com/v1/search`. Results come from `data.results`; OpenClaude reports HTTP and business-code failures separately from an empty result list. A key, when present, is sent only as `Authorization: Bearer <key>` to that endpoint.
|
||||
|
||||
```json
|
||||
{"query":"search terms","max_results":10}
|
||||
```
|
||||
|
||||
### Ollama
|
||||
|
||||
When the active route resolves to Ollama, OpenClaude first calls the configured server's signed-in proxy:
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
import { afterAll, afterEach, describe, expect, test } from 'bun:test'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../../../test/sharedMutationLock.js'
|
||||
import { anysearchProvider } from './anysearch.js'
|
||||
|
||||
await acquireSharedMutationLock('WebSearchTool/providers/anysearch.test.ts')
|
||||
|
||||
const originalFetch = globalThis.fetch
|
||||
const originalKey = process.env.ANYSEARCH_API_KEY
|
||||
const originalMaxResults = process.env.ANYSEARCH_MAX_RESULTS
|
||||
const originalTimeout = process.env.WEB_SEARCH_TIMEOUT_SEC
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch
|
||||
if (originalKey === undefined) delete process.env.ANYSEARCH_API_KEY
|
||||
else process.env.ANYSEARCH_API_KEY = originalKey
|
||||
if (originalMaxResults === undefined) delete process.env.ANYSEARCH_MAX_RESULTS
|
||||
else process.env.ANYSEARCH_MAX_RESULTS = originalMaxResults
|
||||
if (originalTimeout === undefined) delete process.env.WEB_SEARCH_TIMEOUT_SEC
|
||||
else process.env.WEB_SEARCH_TIMEOUT_SEC = originalTimeout
|
||||
})
|
||||
|
||||
afterAll(() => {
|
||||
releaseSharedMutationLock()
|
||||
})
|
||||
|
||||
describe('AnySearch basic search', () => {
|
||||
test('sends an anonymous query and maps the returned search hit', async () => {
|
||||
delete process.env.ANYSEARCH_API_KEY
|
||||
let requestedUrl = ''
|
||||
let requestedInit: RequestInit | undefined
|
||||
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
requestedUrl = String(input)
|
||||
requestedInit = init
|
||||
return new Response(JSON.stringify({
|
||||
code: 0,
|
||||
data: {
|
||||
results: [{
|
||||
title: 'Example',
|
||||
url: 'https://example.com/article',
|
||||
content: 'An example article',
|
||||
}],
|
||||
},
|
||||
}), { status: 200 })
|
||||
}) as typeof fetch
|
||||
|
||||
const output = await anysearchProvider.search({ query: 'example article' })
|
||||
|
||||
expect(requestedUrl).toBe('https://api.anysearch.com/v1/search')
|
||||
expect(requestedInit?.method).toBe('POST')
|
||||
expect(requestedInit?.headers).not.toHaveProperty('Authorization')
|
||||
expect(JSON.parse(String(requestedInit?.body))).toEqual({
|
||||
query: 'example article',
|
||||
max_results: 10,
|
||||
})
|
||||
expect(output.providerName).toBe('anysearch')
|
||||
expect(output.hits).toEqual([{
|
||||
title: 'Example',
|
||||
url: 'https://example.com/article',
|
||||
description: 'An example article',
|
||||
source: 'example.com',
|
||||
}])
|
||||
})
|
||||
|
||||
test('uses an optional key, configured result count, snippet fallback, and domain filters', async () => {
|
||||
process.env.ANYSEARCH_API_KEY = ' test-secret '
|
||||
process.env.ANYSEARCH_MAX_RESULTS = '3'
|
||||
let requestedInit: RequestInit | undefined
|
||||
globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
requestedInit = init
|
||||
return new Response(JSON.stringify({
|
||||
code: 0,
|
||||
data: { results: [
|
||||
{ title: 'Keep', url: 'https://keep.example/a', snippet: 'Kept excerpt' },
|
||||
{ title: 'Drop', url: 'https://drop.example/b', content: 'Dropped excerpt' },
|
||||
] },
|
||||
}), { status: 200 })
|
||||
}) as typeof fetch
|
||||
|
||||
const output = await anysearchProvider.search({
|
||||
query: 'filtered query',
|
||||
blocked_domains: ['drop.example'],
|
||||
})
|
||||
|
||||
expect(requestedInit?.headers).toHaveProperty('Authorization', 'Bearer test-secret')
|
||||
expect(JSON.parse(String(requestedInit?.body))).toEqual({
|
||||
query: 'filtered query',
|
||||
max_results: 3,
|
||||
})
|
||||
expect(output.hits).toEqual([{
|
||||
title: 'Keep',
|
||||
url: 'https://keep.example/a',
|
||||
description: 'Kept excerpt',
|
||||
source: 'keep.example',
|
||||
}])
|
||||
})
|
||||
|
||||
test('treats a blank key as anonymous and ignores an invalid result count', async () => {
|
||||
process.env.ANYSEARCH_API_KEY = ' '
|
||||
process.env.ANYSEARCH_MAX_RESULTS = '11'
|
||||
let requestedInit: RequestInit | undefined
|
||||
globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
requestedInit = init
|
||||
return new Response(JSON.stringify({ code: 0, data: { results: [] } }), { status: 200 })
|
||||
}) as typeof fetch
|
||||
|
||||
const output = await anysearchProvider.search({ query: 'no hits' })
|
||||
|
||||
expect(requestedInit?.headers).not.toHaveProperty('Authorization')
|
||||
expect(JSON.parse(String(requestedInit?.body))).toEqual({ query: 'no hits', max_results: 10 })
|
||||
expect(output.hits).toEqual([])
|
||||
})
|
||||
|
||||
test('reports HTTP auth and quota errors without exposing the key', async () => {
|
||||
process.env.ANYSEARCH_API_KEY = 'private-key'
|
||||
globalThis.fetch = (async (_input: RequestInfo | URL, _init?: RequestInit) => new Response(JSON.stringify({
|
||||
code: 401,
|
||||
message: 'private-key',
|
||||
request_id: 'req-auth',
|
||||
}), { status: 401 })) as typeof fetch
|
||||
|
||||
const authError = await anysearchProvider.search({ query: 'auth' }).catch(error => error)
|
||||
expect(String(authError)).toContain(
|
||||
'AnySearch search error HTTP 401 (check ANYSEARCH_API_KEY) (request_id: req-auth)',
|
||||
)
|
||||
expect(String(authError)).not.toContain('private-key')
|
||||
|
||||
globalThis.fetch = (async (_input: RequestInfo | URL, _init?: RequestInit) => new Response(JSON.stringify({ code: 429 }), {
|
||||
status: 429,
|
||||
})) as typeof fetch
|
||||
await expect(anysearchProvider.search({ query: 'quota' })).rejects.toThrow(
|
||||
/HTTP 429 \(rate limit or quota exceeded\)/,
|
||||
)
|
||||
})
|
||||
|
||||
test('includes a sanitized and truncated snippet for non-JSON HTTP errors', async () => {
|
||||
process.env.ANYSEARCH_API_KEY = 'private-key'
|
||||
const gatewayBody = `gateway failure\nAuthorization: Bearer private-key\n${'x'.repeat(300)}`
|
||||
globalThis.fetch = (async (_input: RequestInfo | URL, _init?: RequestInit) => new Response(gatewayBody, {
|
||||
status: 502,
|
||||
})) as typeof fetch
|
||||
|
||||
const error = await anysearchProvider.search({ query: 'gateway failure' }).catch(error => error)
|
||||
const message = String(error)
|
||||
expect(message).toContain('AnySearch search error HTTP 502')
|
||||
expect(message).toContain('gateway failure')
|
||||
expect(message).toContain('Authorization: [redacted]')
|
||||
expect(message).not.toContain('private-key')
|
||||
expect(message).toContain('…')
|
||||
expect(message).not.toContain('x'.repeat(250))
|
||||
})
|
||||
|
||||
test('separates empty results from business errors and malformed responses', async () => {
|
||||
globalThis.fetch = (async (_input: RequestInfo | URL, _init?: RequestInit) => new Response(JSON.stringify({
|
||||
code: 1001,
|
||||
request_id: 'req-business',
|
||||
data: { results: [] },
|
||||
}), { status: 200 })) as typeof fetch
|
||||
await expect(anysearchProvider.search({ query: 'business error' })).rejects.toThrow(
|
||||
'AnySearch search error code 1001 (request_id: req-business)',
|
||||
)
|
||||
|
||||
globalThis.fetch = (async (_input: RequestInfo | URL, _init?: RequestInit) => new Response(JSON.stringify({
|
||||
code: 0,
|
||||
data: {},
|
||||
}), { status: 200 })) as typeof fetch
|
||||
await expect(anysearchProvider.search({ query: 'missing results' })).rejects.toThrow(
|
||||
'AnySearch search returned an invalid results list',
|
||||
)
|
||||
})
|
||||
|
||||
test('reports a stalled AnySearch request as a timeout', async () => {
|
||||
process.env.WEB_SEARCH_TIMEOUT_SEC = '1'
|
||||
const activeNetwork = setInterval(() => {}, 50)
|
||||
globalThis.fetch = ((_input: RequestInfo | URL, init?: RequestInit) =>
|
||||
new Promise<Response>((_resolve, reject) => {
|
||||
init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true })
|
||||
})) as typeof fetch
|
||||
|
||||
try {
|
||||
await expect(anysearchProvider.search({ query: 'stalled' })).rejects.toThrow(
|
||||
/AnySearch search timed out after 1s/,
|
||||
)
|
||||
} finally {
|
||||
clearInterval(activeNetwork)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,129 @@
|
||||
import type { SearchInput, SearchProvider } from './types.js'
|
||||
import { applyDomainFilters, safeHostname, type ProviderOutput } from './types.js'
|
||||
import { withWebSearchTimeout } from './timeout.js'
|
||||
|
||||
const ANYSEARCH_URL = 'https://api.anysearch.com/v1/search'
|
||||
const DEFAULT_MAX_RESULTS = 10
|
||||
const MAX_ERROR_SNIPPET_LENGTH = 200
|
||||
|
||||
function maxResults(): number {
|
||||
const raw = process.env.ANYSEARCH_MAX_RESULTS?.trim()
|
||||
if (!raw || !/^\d+$/.test(raw)) return DEFAULT_MAX_RESULTS
|
||||
const value = Number(raw)
|
||||
return Number.isInteger(value) && value >= 1 && value <= 10
|
||||
? value
|
||||
: DEFAULT_MAX_RESULTS
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> | undefined {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: undefined
|
||||
}
|
||||
|
||||
function errorDetail(envelope: Record<string, unknown> | undefined): string {
|
||||
const requestId = envelope?.request_id
|
||||
return typeof requestId === 'string' && requestId
|
||||
? ` (request_id: ${requestId})`
|
||||
: ''
|
||||
}
|
||||
|
||||
function errorBodySnippet(rawBody: string): string {
|
||||
const compact = rawBody
|
||||
.replace(/[\u0000-\u001f]+/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
if (!compact) return ''
|
||||
|
||||
const sanitized = compact
|
||||
.replace(
|
||||
/((?:api[_ -]?key|authorization|token|secret|password)\s*[:=]\s*)(?:Bearer\s+)?(?:"[^"]*"|'[^']*'|[^\s,}]+)/gi,
|
||||
'$1[redacted]',
|
||||
)
|
||||
.replace(/Bearer\s+[^\s"',}]+/gi, 'Bearer [redacted]')
|
||||
const clipped = sanitized.slice(0, MAX_ERROR_SNIPPET_LENGTH)
|
||||
return ` (response: ${clipped}${sanitized.length > MAX_ERROR_SNIPPET_LENGTH ? '…' : ''})`
|
||||
}
|
||||
|
||||
export const anysearchProvider: SearchProvider = {
|
||||
name: 'anysearch',
|
||||
|
||||
isConfigured() {
|
||||
return true
|
||||
},
|
||||
|
||||
async search(input: SearchInput, signal?: AbortSignal): Promise<ProviderOutput> {
|
||||
const start = performance.now()
|
||||
const key = process.env.ANYSEARCH_API_KEY?.trim()
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Anysearch-Client': 'openclaude/anysearch-v1',
|
||||
}
|
||||
if (key) headers.Authorization = `Bearer ${key}`
|
||||
|
||||
const { response, body, rawBody } = await withWebSearchTimeout(async combinedSignal => {
|
||||
const response = await fetch(ANYSEARCH_URL, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({ query: input.query, max_results: maxResults() }),
|
||||
signal: combinedSignal,
|
||||
})
|
||||
const rawBody = await response.text()
|
||||
let body: unknown
|
||||
try {
|
||||
body = JSON.parse(rawBody)
|
||||
} catch {
|
||||
body = undefined
|
||||
}
|
||||
return { response, body, rawBody }
|
||||
}, signal, { providerName: 'AnySearch' })
|
||||
|
||||
const envelope = record(body)
|
||||
const detail = errorDetail(envelope)
|
||||
if (!response.ok) {
|
||||
const hint = response.status === 401 || response.status === 403
|
||||
? ' (check ANYSEARCH_API_KEY)'
|
||||
: response.status === 429
|
||||
? ' (rate limit or quota exceeded)'
|
||||
: ''
|
||||
const responseDetail = body === undefined ? errorBodySnippet(rawBody) : ''
|
||||
throw new Error(`AnySearch search error HTTP ${response.status}${hint}${detail}${responseDetail}`)
|
||||
}
|
||||
if (!envelope || typeof envelope.code !== 'number') {
|
||||
throw new Error('AnySearch search returned an invalid response')
|
||||
}
|
||||
if (envelope.code !== 0) {
|
||||
throw new Error(`AnySearch search error code ${envelope.code}${detail}`)
|
||||
}
|
||||
const data = record(envelope.data)
|
||||
if (!Array.isArray(data?.results)) {
|
||||
throw new Error(`AnySearch search returned an invalid results list${detail}`)
|
||||
}
|
||||
|
||||
const hits = data.results.map((raw: unknown) => {
|
||||
const result = record(raw)
|
||||
const title = result?.title
|
||||
const url = result?.url
|
||||
if (typeof title !== 'string' || !title || typeof url !== 'string' || !url) {
|
||||
throw new Error(`AnySearch search returned an invalid result${detail}`)
|
||||
}
|
||||
const description = typeof result.content === 'string' && result.content
|
||||
? result.content
|
||||
: typeof result.snippet === 'string' && result.snippet
|
||||
? result.snippet
|
||||
: undefined
|
||||
return {
|
||||
title,
|
||||
url,
|
||||
description,
|
||||
source: safeHostname(url),
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
hits: applyDomainFilters(hits, input),
|
||||
providerName: 'anysearch',
|
||||
durationSeconds: (performance.now() - start) / 1000,
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -13,6 +13,7 @@ const savedWebSearchEnv = {
|
||||
FIRECRAWL_API_KEY: process.env.FIRECRAWL_API_KEY,
|
||||
FIRECRAWL_API_URL: process.env.FIRECRAWL_API_URL,
|
||||
TAVILY_API_KEY: process.env.TAVILY_API_KEY,
|
||||
ANYSEARCH_API_KEY: process.env.ANYSEARCH_API_KEY,
|
||||
EXA_API_KEY: process.env.EXA_API_KEY,
|
||||
EXA_FREE_TIER: process.env.EXA_FREE_TIER,
|
||||
YOU_API_KEY: process.env.YOU_API_KEY,
|
||||
@@ -128,6 +129,11 @@ describe('getProviderMode', () => {
|
||||
expect(getProviderMode()).toBe('ollama')
|
||||
})
|
||||
|
||||
test('returns anysearch mode', () => {
|
||||
process.env.WEB_SEARCH_PROVIDER = 'anysearch'
|
||||
expect(getProviderMode()).toBe('anysearch')
|
||||
})
|
||||
|
||||
test('returns native mode', () => {
|
||||
process.env.WEB_SEARCH_PROVIDER = 'native'
|
||||
expect(getProviderMode()).toBe('native')
|
||||
@@ -156,6 +162,19 @@ describe('getProviderChain', () => {
|
||||
expect(chain.some(p => p.name === 'custom')).toBe(false)
|
||||
})
|
||||
|
||||
test('auto mode does not select AnySearch without an explicit choice', () => {
|
||||
const chain = getProviderChain('auto')
|
||||
expect(chain.some(p => p.name === 'anysearch')).toBe(false)
|
||||
})
|
||||
|
||||
test('anysearch mode selects the built-in provider without a key', () => {
|
||||
delete process.env.ANYSEARCH_API_KEY
|
||||
const chain = getProviderChain('anysearch' as ProviderMode)
|
||||
expect(chain).toHaveLength(1)
|
||||
expect(chain[0].name).toBe('anysearch')
|
||||
expect(chain[0].isConfigured()).toBe(true)
|
||||
})
|
||||
|
||||
test('custom mode explicitly returns custom provider', () => {
|
||||
const chain = getProviderChain('custom' as ProviderMode)
|
||||
expect(chain).toHaveLength(1)
|
||||
@@ -194,6 +213,19 @@ describe('getProviderChain', () => {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('runSearch', () => {
|
||||
test('explicit AnySearch failure does not fall back to another provider', async () => {
|
||||
process.env.WEB_SEARCH_PROVIDER = 'anysearch'
|
||||
let requestCount = 0
|
||||
globalThis.fetch = (async (_input: RequestInfo | URL, _init?: RequestInit) => {
|
||||
requestCount += 1
|
||||
return new Response(JSON.stringify({ code: 429 }), { status: 429 })
|
||||
}) as typeof fetch
|
||||
|
||||
const { runSearch } = await import('./index.js')
|
||||
await expect(runSearch({ query: 'quota' })).rejects.toThrow(/AnySearch search error HTTP 429/)
|
||||
expect(requestCount).toBe(1)
|
||||
})
|
||||
|
||||
test('AbortError stops the chain immediately in auto mode', async () => {
|
||||
// Use AbortController to cancel
|
||||
const controller = new AbortController()
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
* "ollama" — use Ollama local/hosted Web Search API only (fail loudly)
|
||||
* "firecrawl" — use Firecrawl only (fail loudly)
|
||||
* "tavily" — use Tavily only (fail loudly)
|
||||
* "anysearch" — use AnySearch only, with an optional API key (fail loudly)
|
||||
* "exa" — use Exa only (fail loudly): the keyed API when EXA_API_KEY
|
||||
* is set, otherwise the keyless free tier
|
||||
* "you" — use You.com only (fail loudly)
|
||||
@@ -33,6 +34,7 @@ import { customProvider } from './custom.js'
|
||||
import { duckduckgoProvider } from './duckduckgo.js'
|
||||
import { firecrawlProvider } from './firecrawl.js'
|
||||
import { tavilyProvider } from './tavily.js'
|
||||
import { anysearchProvider } from './anysearch.js'
|
||||
import { exaProvider } from './exa.js'
|
||||
import { exaFreeProvider } from './exaFree.js'
|
||||
import { youProvider } from './you.js'
|
||||
@@ -93,6 +95,7 @@ export type ProviderMode =
|
||||
| 'firecrawl'
|
||||
| 'ddg'
|
||||
| 'tavily'
|
||||
| 'anysearch'
|
||||
| 'exa'
|
||||
| 'you'
|
||||
| 'jina'
|
||||
@@ -108,6 +111,7 @@ const PROVIDER_BY_NAME: Record<string, SearchProvider> = {
|
||||
firecrawl: firecrawlProvider,
|
||||
ddg: duckduckgoProvider,
|
||||
tavily: tavilyProvider,
|
||||
anysearch: anysearchProvider,
|
||||
exa: exaProvider,
|
||||
you: youProvider,
|
||||
jina: jinaProvider,
|
||||
|
||||
Reference in New Issue
Block a user