From f235c835d14e1fa9ba200f5b1df3e50a2da7f3c4 Mon Sep 17 00:00:00 2001 From: JATMN Date: Thu, 29 Oct 2026 16:54:40 -0700 Subject: [PATCH] feat(web-fetch): add opt-in AI/ML API Toolbox scraping with restored test state (#2468) * feat(web-fetch): add an opt-in AI/ML API Toolbox scraping backend WebFetch can already replace its raw HTTP fetch with Firecrawl's scrape endpoint. This adds the AI/ML API Toolbox as a second scraping backend, selected with WEB_FETCH_PROVIDER=aimlapi and authenticated with the existing AIMLAPI_API_KEY. Toolbox runs are billed per page, so the backend is opt-in only: an AI/ML API key on its own never changes how WebFetch fetches pages. When selected it takes precedence over a Firecrawl key. - Runs the search1api/crawl tool, chosen by measurement against the other page-to-Markdown tools ($0.0013 per run, main-content Markdown with links, unbilled is_error runs for missing pages), with max_cost_usd capped at $0.005. - Polls queued (202) runs, reports is_error runs and over-cost refusals as WebFetch errors, and retries 429/502/503/504 with the same idempotency key so a retry is never charged twice. - Sends the key and the X-AIMLAPI-Source / X-AIMLAPI-Partner-ID attribution headers only to the fixed tools.aimlapi.com endpoint. - Like the Firecrawl path, a failed scrape is reported rather than retried over plain HTTP. * fix(web-fetch): never send an aimlapi proxy profile's key to the Toolbox An aimlapi provider profile that points at the user's own proxy exports the proxy's credential as AIMLAPI_API_KEY. The WebFetch Toolbox backend now treats that key as not configured when the route is aimlapi and OPENAI_BASE_URL (or OPENAI_API_BASE) is not the canonical https://api.aimlapi.com/v1, so it is never sent to tools.aimlapi.com. * fix(web-fetch): match the search backend's credential and redirect handling Apply the same hardening #2465 gave the AI/ML API web search backend: - send only the first usable key of a comma-separated AIMLAPI_API_KEY pool (firstUsableCredential), instead of the whole list as one Bearer token; - treat the OPENAI_API_BASE alias like OPENAI_BASE_URL when deciding that an aimlapi profile points at a proxy, so its credential is withheld; - send both Toolbox requests with redirect: 'error', so the key and the attribution headers never reach another host; - redact the active key from Toolbox error text before it is shown. Tests cover each case. * docs(web-fetch): describe the requested cost limit and first-key pool use Word the $0.005 max_cost_usd as a limit each run requests, as the merged search docs do, and note that a pooled AIMLAPI_API_KEY contributes only its first usable key. * docs(web-fetch): document the alias, pool and redirect rules in the setup guide Describe the requested $0.005 cost limit, the OPENAI_API_BASE alias in the proxy check, first-key use of a pooled AIMLAPI_API_KEY, and that Toolbox requests never follow redirects. * test(web-fetch): keep the backend test's model stub and env changes local Replace the process-global mock.module() of services/api/claude.js with a spyOn() on queryHaiku, which mock.restore() removes, so test files that run later in the same process get the real queryHaiku again. Calls are counted on a local stub, since the spy reuses an earlier file's leftover module mock. Snapshot the fixture's variables with snapshotEnv() after taking the shared mutation lock and clear them, including CLAUDE_CODE_PROVIDER_ROUTE_ID, OPENAI_BASE_URL and OPENAI_API_BASE, so an ambient aimlapi proxy profile no longer withholds the test key. They are restored before the lock is released. * fix(env-file): accept AIMLAPI_API_KEY in --provider-env-file .env.example lists AIMLAPI_API_KEY for the file-based setup, and the AI/ML API WebFetch backend pairs it with WEB_FETCH_PROVIDER, but the loader rejected the key as unsupported. Allow it, and cover loading both values. * test(web-fetch): restore inherited model mock after backend calls * docs(web-fetch): clarify retry scope and external URL disclosure --------- Co-authored-by: Stan Co-authored-by: Hugo --- .env.example | 7 + README.md | 9 + docs/aimlapi-setup.md | 35 ++ .../WebFetchTool/WebFetchTool.backend.test.ts | 141 ++++++++ .../WebFetchTool.backendIsolation.test.ts | 43 +++ src/tools/WebFetchTool/WebFetchTool.ts | 23 +- src/tools/WebFetchTool/aimlapiScrape.test.ts | 325 ++++++++++++++++++ src/tools/WebFetchTool/aimlapiScrape.ts | 302 ++++++++++++++++ src/utils/envFile.test.ts | 17 + src/utils/envFile.ts | 2 + 10 files changed, 902 insertions(+), 2 deletions(-) create mode 100644 src/tools/WebFetchTool/WebFetchTool.backend.test.ts create mode 100644 src/tools/WebFetchTool/WebFetchTool.backendIsolation.test.ts create mode 100644 src/tools/WebFetchTool/aimlapiScrape.test.ts create mode 100644 src/tools/WebFetchTool/aimlapiScrape.ts diff --git a/.env.example b/.env.example index 69ddd0dcb..9861782f0 100644 --- a/.env.example +++ b/.env.example @@ -573,6 +573,13 @@ ANTHROPIC_API_KEY=sk-ant-your-key-here # Self-hosted Firecrawl endpoint (optional — omit to use cloud API) # FIRECRAWL_API_URL=https://your-firecrawl-instance.com +# WebFetch page scraping through the AI/ML API Toolbox (opt-in, paid per page: +# about $0.0013 a run; each run requests a $0.005 cost limit). Uses +# AIMLAPI_API_KEY (the first key of a comma-separated pool) and takes +# precedence over a Firecrawl key for WebFetch. AIMLAPI_API_KEY alone does not +# turn it on. +# WEB_FETCH_PROVIDER=aimlapi + # Ollama Web Search. A signed-in local Ollama route needs no key; the key # enables https://ollama.com/api/web_search as a hosted fallback. # OLLAMA_BASE_URL=http://localhost:11434 diff --git a/README.md b/README.md index b2a16747f..9d2636276 100644 --- a/README.md +++ b/README.md @@ -508,6 +508,15 @@ With Firecrawl enabled: Free tier at [firecrawl.dev](https://firecrawl.dev) includes 500 credits. The key is optional. +`WebFetch` can also scrape pages through the [AI/ML API](https://aimlapi.com) Toolbox with your AI/ML API key. Each fetched page is one paid Toolbox run (about $0.0013; each run requests a $0.005 cost limit), so it is used only when you choose it: + +```bash +export AIMLAPI_API_KEY=your-aimlapi-key +export WEB_FETCH_PROVIDER=aimlapi +``` + +`WEB_FETCH_PROVIDER=aimlapi` takes precedence over a Firecrawl key for `WebFetch`. Having `AIMLAPI_API_KEY` set without it does not change how pages are fetched. See [AI/ML API Setup](docs/aimlapi-setup.md#webfetch-through-the-aiml-api-toolbox). + ## Headless gRPC Server OpenClaude can run as a headless gRPC service with bidirectional streaming — diff --git a/docs/aimlapi-setup.md b/docs/aimlapi-setup.md index 9197d7c37..5c13b4940 100644 --- a/docs/aimlapi-setup.md +++ b/docs/aimlapi-setup.md @@ -86,6 +86,41 @@ for the request and response format. - `/model` lists chat-capable models discovered from the catalog. - Send any prompt to confirm responses come back from the selected model. +## WebFetch through the AI/ML API Toolbox + +`WebFetch` normally fetches pages over plain HTTP and converts the HTML to +Markdown, which can miss JavaScript-rendered content. With an AI/ML API key you +can have it scrape pages through the +[AI/ML API Toolbox](https://docs.aimlapi.com/quickstart/toolbox) instead: + +```bash +export AIMLAPI_API_KEY="your-aimlapi-key" +export WEB_FETCH_PROVIDER=aimlapi +``` + +- **Opt-in.** Each fetched page is one Toolbox run of `search1api/crawl`, billed + to your AI/ML API balance (about $0.0013 per page on 2026-10-19). Every run + requests a cost limit with `max_cost_usd: 0.005`. An AI/ML API key on its own + never turns this on. +- **URL disclosure.** The full requested URL, including query parameters, is + sent to the external AI/ML API Toolbox. Do not use this backend for private, + internal, or token-bearing URLs. +- `WEB_FETCH_PROVIDER=aimlapi` takes precedence over `FIRECRAWL_API_KEY` / + `FIRECRAWL_API_URL` for `WebFetch`. `WebSearch` is not affected. +- Like the Firecrawl backend, a failed scrape (a page that does not exist, a + refused or failed run) is reported as a `WebFetch` error instead of being + retried over plain HTTP. Within one `WebFetch` call, retries of a run request + reuse the same input and idempotency key. +- The key and the attribution headers below are sent only to + `https://tools.aimlapi.com`, which is fixed and cannot be overridden. When + the active aimlapi profile points at your own proxy (a non-canonical + `OPENAI_BASE_URL` or `OPENAI_API_BASE`), its key is treated as the proxy's + and is not sent; set up a canonical AI/ML API key to use this backend. When + `AIMLAPI_API_KEY` holds a comma-separated pool, only its first usable key is + sent, with no rotation across keys. Requests never follow redirects. A + scraped page is untrusted web content and goes through the same `WebFetch` + processing as a page fetched over plain HTTP. + ## Notes - Model discovery uses the public, unauthenticated `GET /models` endpoint and surfaces only chat-completions models; image, audio, embeddings, and other modalities are intentionally not routed through the coding workflow. diff --git a/src/tools/WebFetchTool/WebFetchTool.backend.test.ts b/src/tools/WebFetchTool/WebFetchTool.backend.test.ts new file mode 100644 index 000000000..bc70d1624 --- /dev/null +++ b/src/tools/WebFetchTool/WebFetchTool.backend.test.ts @@ -0,0 +1,141 @@ +import { afterEach, beforeEach, expect, mock, spyOn, test } from 'bun:test' + +import * as claudeApi from '../../services/api/claude.js' +import { snapshotEnv } from '../../test/envSnapshot.js' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../../test/sharedMutationLock.js' +import { asMockFetch } from '../../test/typedMocks.js' +import { createAssistantMessage } from '../../utils/messages/factories.js' + +// WebFetch hands scraped markdown to the secondary model. Each test routes +// queryHaiku to this stub only for the duration of the WebFetch call. A spy +// can reuse an earlier file's mock, so preserve that mock's implementation +// and count calls on our own stub. +const haikuStub = mock(async ({ userPrompt }: { userPrompt: string }) => + createAssistantMessage({ + content: `summary of: ${userPrompt.includes('# Scraped page') ? 'scraped page' : 'other'}`, + }), +) +const fetchMock = mock() +const originalFetch = globalThis.fetch +// The route and both base-URL spellings decide whether AIMLAPI_API_KEY may be +// sent: an ambient aimlapi proxy profile would withhold the test key, so they +// are cleared along with the backend selectors. +const ENV_KEYS = [ + 'AIMLAPI_API_KEY', + 'FIRECRAWL_API_KEY', + 'FIRECRAWL_API_URL', + 'WEB_FETCH_PROVIDER', + 'CLAUDE_CODE_PROVIDER_ROUTE_ID', + 'OPENAI_BASE_URL', + 'OPENAI_API_BASE', +] as const +let restoreEnv: (() => void) | undefined + +beforeEach(async () => { + await acquireSharedMutationLock('tools/WebFetchTool/WebFetchTool.backend.test.ts') + restoreEnv = snapshotEnv(ENV_KEYS, { clear: true }) + haikuStub.mockClear() + fetchMock.mockReset() + fetchMock.mockImplementation(async (input: unknown) => { + const host = new URL(String(input)).hostname + if (host === 'tools.aimlapi.com') { + return Response.json({ + id: 'run-1', + status: 'completed', + is_error: false, + output: { results: { content: '# Scraped page' } }, + cost: { quoted_usd: 0.0013, actual_usd: 0.0013, over_max: false }, + }) + } + if (host === 'api.firecrawl.dev') { + return Response.json({ success: true, data: { markdown: '# Scraped page' } }) + } + throw new Error(`unexpected request to ${host}`) + }) + globalThis.fetch = asMockFetch(fetchMock) +}) + +afterEach(() => { + try { + globalThis.fetch = originalFetch + restoreEnv?.() + restoreEnv = undefined + } finally { + releaseSharedMutationLock() + } +}) + +async function runWebFetch(url = 'https://example.com/page') { + const nonce = `${Date.now()}-${Math.random()}` + const { WebFetchTool } = await import(`./WebFetchTool.js?ts=${nonce}`) + const previousQueryHaiku = claudeApi.queryHaiku + const haikuSpy = spyOn(claudeApi, 'queryHaiku') + try { + let result!: Awaited> + await haikuSpy.withImplementation(haikuStub, async () => { + result = await WebFetchTool.call( + { url, prompt: 'summarize' }, + { + abortController: new AbortController(), + options: { isNonInteractiveSession: true }, + } as Parameters[1], + ) + }) + return result + } finally { + // withImplementation restores inherited mocks, including on rejection. + // Only remove the spy when this call created it. + if (haikuSpy !== previousQueryHaiku) haikuSpy.mockRestore() + } +} + +function requestedHosts(): string[] { + return fetchMock.mock.calls.map(call => new URL(String(call[0])).hostname) +} + +test('an AI/ML API key alone keeps WebFetch on the Firecrawl backend', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + process.env.FIRECRAWL_API_KEY = 'fc-test-key' + + const { data } = await runWebFetch() + + expect(requestedHosts()).toEqual(['api.firecrawl.dev']) + expect(data.result).toBe('summary of: scraped page') +}) + +test('WEB_FETCH_PROVIDER=aimlapi scrapes through the AI/ML API Toolbox, ahead of Firecrawl', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + process.env.FIRECRAWL_API_KEY = 'fc-test-key' + process.env.WEB_FETCH_PROVIDER = 'aimlapi' + + const { data } = await runWebFetch() + + expect(requestedHosts()).toEqual(['tools.aimlapi.com']) + expect(data).toMatchObject({ + bytes: Buffer.byteLength('# Scraped page'), + code: 200, + result: 'summary of: scraped page', + url: 'https://example.com/page', + }) +}) + +test('an AI/ML API failure is reported, not retried through another backend', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + process.env.FIRECRAWL_API_KEY = 'fc-test-key' + process.env.WEB_FETCH_PROVIDER = 'aimlapi' + fetchMock.mockImplementation(async () => + Response.json({ + id: 'run-2', + status: 'completed', + is_error: true, + error: { kind: 'provider_error', retryable: false, message: 'The provider answered HTTP 400.' }, + }), + ) + + await expect(runWebFetch()).rejects.toThrow('The provider answered HTTP 400.') + expect(requestedHosts()).toEqual(['tools.aimlapi.com']) + expect(haikuStub).not.toHaveBeenCalled() +}) diff --git a/src/tools/WebFetchTool/WebFetchTool.backendIsolation.test.ts b/src/tools/WebFetchTool/WebFetchTool.backendIsolation.test.ts new file mode 100644 index 000000000..3a4816f2e --- /dev/null +++ b/src/tools/WebFetchTool/WebFetchTool.backendIsolation.test.ts @@ -0,0 +1,43 @@ +import { test } from 'bun:test' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + REAL_PROVIDER_TEST_TIMEOUT_MS, + runTestFileWithRealProviders, +} from '../../test/providerModuleIsolation.js' + +for (const inheritedMock of [false, true]) { + test(`backend tests restore queryHaiku (inherited mock: ${inheritedMock})`, async () => { + const root = await mkdtemp(join(tmpdir(), 'webfetch-backend-isolation-')) + const entry = join(root, 'isolation.test.ts') + try { + // Exercise the actual suite's setup, calls and teardown in a disposable + // process so this deliberately inherited module mock cannot leak either. + await writeFile(entry, ` + import { afterEach, expect, mock } from 'bun:test' + import * as claudeApi from ${JSON.stringify(join(import.meta.dir, '../../services/api/claude.ts'))} + const inheritedMock = ${inheritedMock} + const original = inheritedMock ? mock(async () => 'prior response') : claudeApi.queryHaiku + if (inheritedMock) { + mock.module(${JSON.stringify(join(import.meta.dir, '../../services/api/claude.ts'))}, () => ({ + ...claudeApi, queryHaiku: original, + })) + } + await import(${JSON.stringify(join(import.meta.dir, 'WebFetchTool.backend.test.ts'))}) + afterEach(async () => { + expect(claudeApi.queryHaiku).toBe(original) + if (inheritedMock) { + expect(await claudeApi.queryHaiku({ userPrompt: 'after backend tests' })).toBe('prior response') + } else { + expect('mock' in claudeApi.queryHaiku).toBe(false) + } + }) + `) + await runTestFileWithRealProviders(entry) + } finally { + await rm(root, { recursive: true, force: true }) + } + // Allow the child deadline plus time for assertions and fixture cleanup. + }, { timeout: REAL_PROVIDER_TEST_TIMEOUT_MS + 5_000 }) +} diff --git a/src/tools/WebFetchTool/WebFetchTool.ts b/src/tools/WebFetchTool/WebFetchTool.ts index 86b1e7956..48854466a 100644 --- a/src/tools/WebFetchTool/WebFetchTool.ts +++ b/src/tools/WebFetchTool/WebFetchTool.ts @@ -22,6 +22,7 @@ import { MAX_MARKDOWN_LENGTH, } from './utils.js' import { firecrawlScrape } from '../firecrawl/client.js' +import { isAimlapiWebFetchSelected, scrapeWithAimlapi } from './aimlapiScrape.js' function isFirecrawlEnabled(): boolean { return Boolean(process.env.FIRECRAWL_API_KEY) || Boolean(process.env.FIRECRAWL_API_URL) @@ -41,6 +42,23 @@ async function scrapeWithFirecrawl( return { markdown, bytes: Buffer.byteLength(markdown) } } +type PageScraper = ( + url: string, + signal?: AbortSignal, +) => Promise<{ markdown: string; bytes: number }> + +/** + * The scraping backend that replaces raw HTTP, if any. AI/ML API runs are + * billed per page, so that backend needs the explicit WEB_FETCH_PROVIDER=aimlapi + * choice and then wins over a configured Firecrawl key. + */ +function getPageScraper(): PageScraper | null { + if (isAimlapiWebFetchSelected()) { + return (url, signal) => scrapeWithAimlapi(url, { signal }) + } + return isFirecrawlEnabled() ? scrapeWithFirecrawl : null +} + const inputSchema = lazySchema(() => z.strictObject({ url: z.url().describe('The URL to fetch content from'), @@ -231,8 +249,9 @@ ${DESCRIPTION}` ) { const start = Date.now() - if (isFirecrawlEnabled()) { - const { markdown, bytes } = await scrapeWithFirecrawl(url, abortController.signal) + const scrapePage = getPageScraper() + if (scrapePage) { + const { markdown, bytes } = await scrapePage(url, abortController.signal) const result = await applyPromptToMarkdown( prompt, markdown, diff --git a/src/tools/WebFetchTool/aimlapiScrape.test.ts b/src/tools/WebFetchTool/aimlapiScrape.test.ts new file mode 100644 index 000000000..4c1b4a289 --- /dev/null +++ b/src/tools/WebFetchTool/aimlapiScrape.test.ts @@ -0,0 +1,325 @@ +import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test' + +import { asMockFetch, callArgs } from '../../test/typedMocks.js' +import { + AIMLAPI_WEB_FETCH_MAX_COST_USD, + AIMLAPI_WEB_FETCH_TOOL_ID, + isAimlapiWebFetchSelected, + scrapeWithAimlapi, +} from './aimlapiScrape.js' + +const originalFetch = globalThis.fetch +const originalEnv = { + AIMLAPI_API_KEY: process.env.AIMLAPI_API_KEY, + WEB_FETCH_PROVIDER: process.env.WEB_FETCH_PROVIDER, + CLAUDE_CODE_PROVIDER_ROUTE_ID: process.env.CLAUDE_CODE_PROVIDER_ROUTE_ID, + OPENAI_BASE_URL: process.env.OPENAI_BASE_URL, + OPENAI_API_BASE: process.env.OPENAI_API_BASE, +} + +function restoreEnv(key: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[key] + } else { + process.env[key] = value + } +} + +beforeEach(() => { + // An ambient aimlapi proxy profile would withhold the key from every test. + delete process.env.CLAUDE_CODE_PROVIDER_ROUTE_ID + delete process.env.OPENAI_BASE_URL + delete process.env.OPENAI_API_BASE +}) + +afterEach(() => { + globalThis.fetch = originalFetch + for (const [key, value] of Object.entries(originalEnv)) restoreEnv(key, value) +}) + +function json(body: unknown, status = 200, headers: Record = {}): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json', ...headers }, + }) +} + +function completedRun(content: string): Record { + return { + id: 'run-1', + tool: AIMLAPI_WEB_FETCH_TOOL_ID, + status: 'completed', + is_error: false, + untrusted: true, + output: { + crawlParameters: { url: 'https://example.com' }, + results: { title: 'Example', link: 'https://example.com', content }, + }, + cost: { quoted_usd: 0.0013, actual_usd: 0.0013, credits: 2600, over_max: false }, + } +} + +function requestBody(fetchMock: Parameters[0], index: number): Record { + const init = callArgs(fetchMock, index)[1] as RequestInit + return JSON.parse(String(init.body)) as Record +} + +describe('AI/ML API WebFetch backend selection', () => { + test('an AI/ML API key alone does not select the paid backend', () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + delete process.env.WEB_FETCH_PROVIDER + + expect(isAimlapiWebFetchSelected()).toBe(false) + }) + + test('WEB_FETCH_PROVIDER=aimlapi selects it, ignoring case and spaces', () => { + process.env.WEB_FETCH_PROVIDER = ' AIMLAPI ' + expect(isAimlapiWebFetchSelected()).toBe(true) + + process.env.WEB_FETCH_PROVIDER = 'firecrawl' + expect(isAimlapiWebFetchSelected()).toBe(false) + }) +}) + +describe('scrapeWithAimlapi', () => { + test('runs the pinned tool with a cost cap, bearer auth and attribution headers', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + const fetchMock = mock(async (input: unknown, init?: RequestInit) => { + expect(String(input)).toBe('https://tools.aimlapi.com/v1/tools/run') + expect(init?.method).toBe('POST') + expect(init?.redirect).toBe('error') + const headers = init?.headers as Record + expect(headers.Authorization).toBe('Bearer aiml-test-key') + expect(headers['X-AIMLAPI-Source']).toBe('agent/openclaude') + expect(headers['X-AIMLAPI-Partner-ID']).toBe('part_62yQoGYDq4Yqnrj2R1iGrDNJ') + return json(completedRun('# Héllo')) + }) + globalThis.fetch = asMockFetch(fetchMock) + + await expect(scrapeWithAimlapi('https://example.com')).resolves.toEqual({ + markdown: '# Héllo', + bytes: Buffer.byteLength('# Héllo'), + }) + + const body = requestBody(fetchMock, 0) + expect(body).toMatchObject({ + tool: 'search1api/crawl', + input: { url: 'https://example.com' }, + max_cost_usd: AIMLAPI_WEB_FETCH_MAX_COST_USD, + }) + expect(typeof body.idempotency_key).toBe('string') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + test('requires an AI/ML API key and sends nothing without one', async () => { + delete process.env.AIMLAPI_API_KEY + const fetchMock = mock(async () => json(completedRun('unused'))) + globalThis.fetch = asMockFetch(fetchMock) + + await expect(scrapeWithAimlapi('https://example.com')).rejects.toThrow( + 'WEB_FETCH_PROVIDER=aimlapi needs an AI/ML API key', + ) + expect(fetchMock).not.toHaveBeenCalled() + }) + + test('does not send a key exported by an aimlapi profile that points at a proxy', async () => { + process.env.AIMLAPI_API_KEY = 'proxy-credential' + process.env.CLAUDE_CODE_PROVIDER_ROUTE_ID = 'aimlapi' + process.env.OPENAI_BASE_URL = 'https://my-proxy.example.com/v1' + const fetchMock = mock(async () => json(completedRun('unused'))) + globalThis.fetch = asMockFetch(fetchMock) + + await expect(scrapeWithAimlapi('https://example.com')).rejects.toThrow( + 'WEB_FETCH_PROVIDER=aimlapi needs an AI/ML API key', + ) + expect(fetchMock).not.toHaveBeenCalled() + }) + + test('uses the key of an aimlapi profile on the canonical endpoint', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-profile-key' + process.env.CLAUDE_CODE_PROVIDER_ROUTE_ID = 'aimlapi' + process.env.OPENAI_BASE_URL = 'https://api.aimlapi.com/v1' + const fetchMock = mock(async (_input: unknown, init?: RequestInit) => { + expect((init?.headers as Record).Authorization).toBe('Bearer aiml-profile-key') + return json(completedRun('# Canonical')) + }) + globalThis.fetch = asMockFetch(fetchMock) + + await expect(scrapeWithAimlapi('https://example.com')).resolves.toMatchObject({ + markdown: '# Canonical', + }) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + test('does not send a proxy profile key when only the OPENAI_API_BASE alias is set', async () => { + process.env.AIMLAPI_API_KEY = 'proxy-credential' + process.env.CLAUDE_CODE_PROVIDER_ROUTE_ID = 'aimlapi' + process.env.OPENAI_API_BASE = 'https://my-proxy.example.com/v1' + const fetchMock = mock(async () => json(completedRun('unused'))) + globalThis.fetch = asMockFetch(fetchMock) + + await expect(scrapeWithAimlapi('https://example.com')).rejects.toThrow( + 'WEB_FETCH_PROVIDER=aimlapi needs an AI/ML API key', + ) + expect(fetchMock).not.toHaveBeenCalled() + }) + + test('sends only the first key of a pooled AIMLAPI_API_KEY', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-first-key, aiml-second-key' + const fetchMock = mock(async (_input: unknown, init?: RequestInit) => { + expect((init?.headers as Record).Authorization).toBe('Bearer aiml-first-key') + return json(completedRun('# Pooled')) + }) + globalThis.fetch = asMockFetch(fetchMock) + + await expect(scrapeWithAimlapi('https://example.com')).resolves.toMatchObject({ + markdown: '# Pooled', + }) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + test('refuses a redirect, so the key and attribution never leave the Toolbox host', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + const fetchMock = mock(async (_input: unknown, init?: RequestInit) => { + // What fetch does with a 3xx response when redirect is 'error'. + if (init?.redirect === 'error') throw new TypeError('unexpected redirect') + return new Response(null, { status: 302, headers: { Location: 'https://elsewhere.example/' } }) + }) + globalThis.fetch = asMockFetch(fetchMock) + + await expect(scrapeWithAimlapi('https://example.com')).rejects.toThrow('unexpected redirect') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + test('redacts the key when the Toolbox echoes it in an error', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-echoed-key' + globalThis.fetch = asMockFetch(mock(async () => + json({ error: { code: 'invalid_api_key', message: 'Key aiml-echoed-key is not valid.' } }, 401), + )) + + const error = String(await scrapeWithAimlapi('https://example.com').catch(e => e)) + expect(error).toContain('AI/ML API Toolbox run error 401 (invalid_api_key): Key [redacted] is not valid.') + expect(error).not.toContain('aiml-echoed-key') + }) + + test('polls a queued (202) run until it completes', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + const responses = [ + json({ id: 'run 7', status: 'queued', next_poll_hint_ms: 1 }, 202), + json({ id: 'run 7', status: 'running', next_poll_hint_ms: 1 }), + json({ ...completedRun('# Done'), id: 'run 7' }), + ] + const fetchMock = mock(async () => responses.shift()!) + globalThis.fetch = asMockFetch(fetchMock) + + await expect(scrapeWithAimlapi('https://example.com')).resolves.toMatchObject({ + markdown: '# Done', + }) + + expect(fetchMock).toHaveBeenCalledTimes(3) + const [pollUrl, pollInit] = callArgs(fetchMock, 1) as [string, RequestInit] + expect(pollUrl).toBe('https://tools.aimlapi.com/v1/tools/runs/run%207') + expect(pollInit.method).toBe('GET') + expect(pollInit.redirect).toBe('error') + expect((pollInit.headers as Record)['X-AIMLAPI-Partner-ID']).toBe( + 'part_62yQoGYDq4Yqnrj2R1iGrDNJ', + ) + }) + + test('reports an is_error run with its message instead of returning content', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + globalThis.fetch = asMockFetch(mock(async () => + json({ + id: 'run-2', + status: 'completed', + is_error: true, + error: { + kind: 'no_results', + retryable: false, + message: 'The provider found no data for this input (HTTP 404).', + }, + cost: { quoted_usd: 0.0013, actual_usd: 0, credits: 0, over_max: false }, + }), + )) + + await expect(scrapeWithAimlapi('https://example.com/missing')).rejects.toThrow( + 'search1api/crawl failed for https://example.com/missing: The provider found no data for this input (HTTP 404).', + ) + }) + + test('reports a run that ended in a failed status', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + globalThis.fetch = asMockFetch(mock(async () => + json({ id: 'run-3', status: 'timed_out', cost: { over_max: true } }), + )) + + await expect(scrapeWithAimlapi('https://example.com')).rejects.toThrow( + 'run ended with status timed_out. The run hit the $0.005 max_cost_usd cap.', + ) + }) + + test('surfaces an over-cost refusal without retrying it', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + const fetchMock = mock(async () => + json( + { + error: { + message: 'The cheapest source costs 0.0013 USD, above the limit of 0.001 USD.', + type: 'invalid_request_error', + code: 'max_cost_exceeded', + }, + }, + 400, + ), + ) + globalThis.fetch = asMockFetch(fetchMock) + + await expect(scrapeWithAimlapi('https://example.com')).rejects.toThrow( + 'AI/ML API Toolbox run error 400 (max_cost_exceeded): The cheapest source costs 0.0013 USD, above the limit of 0.001 USD.', + ) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + test('retries a rate-limited run with the same idempotency key', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + const responses = [ + json({ error: { code: 'rate_limited', message: 'Too many requests' } }, 429), + json(completedRun('# After retry')), + ] + const fetchMock = mock(async () => responses.shift()!) + globalThis.fetch = asMockFetch(fetchMock) + + await expect( + scrapeWithAimlapi('https://example.com', { backoffMs: 1 }), + ).resolves.toMatchObject({ markdown: '# After retry' }) + + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(requestBody(fetchMock, 1).idempotency_key).toBe( + requestBody(fetchMock, 0).idempotency_key, + ) + }) + + test('gives up on rate limiting after the last attempt', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + const fetchMock = mock(async () => + json({ error: { code: 'rate_limited', message: 'Too many requests' } }, 429), + ) + globalThis.fetch = asMockFetch(fetchMock) + + await expect( + scrapeWithAimlapi('https://example.com', { backoffMs: 1, maxAttempts: 2 }), + ).rejects.toThrow('AI/ML API Toolbox run error 429 (rate_limited): Too many requests') + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + + test('rejects a completed run without page content', async () => { + process.env.AIMLAPI_API_KEY = 'aiml-test-key' + globalThis.fetch = asMockFetch(mock(async () => + json({ ...completedRun(''), output: { results: {} } }), + )) + + await expect(scrapeWithAimlapi('https://example.com')).rejects.toThrow( + 'search1api/crawl returned no page content for https://example.com.', + ) + }) +}) diff --git a/src/tools/WebFetchTool/aimlapiScrape.ts b/src/tools/WebFetchTool/aimlapiScrape.ts new file mode 100644 index 000000000..3ea29f51a --- /dev/null +++ b/src/tools/WebFetchTool/aimlapiScrape.ts @@ -0,0 +1,302 @@ +/** + * AI/ML API Toolbox backend for WebFetch: fetches a page as Markdown through a + * Toolbox tool run instead of raw HTTP, the same role Firecrawl's scrape + * endpoint plays. + * + * Every run is billed to the user's AI/ML API balance, so this backend is + * opt-in only: it runs when WEB_FETCH_PROVIDER=aimlapi is set, never just + * because an AI/ML API key is present. + */ + +import { randomUUID } from 'node:crypto' + +import { + AIMLAPI_SOURCE, + isCanonicalAimlapiInferenceBaseUrl, + PARTNER_HEADER_NAME, + resolvePartnerId, + SOURCE_HEADER_NAME, +} from '../../integrations/aimlapi/config.js' +import { sleep } from '../../integrations/aimlapi/transport.js' +import { firstUsableCredential } from '../../services/api/credentialPool.js' +import { createCombinedAbortSignal } from '../../utils/combinedAbortSignal.js' + +/** + * Fixed and not user-overridable, so the AI/ML API key and the attribution + * headers can only ever be sent to tools.aimlapi.com. + */ +export const AIMLAPI_TOOLBOX_BASE_URL = 'https://tools.aimlapi.com/v1/tools' + +/** + * The Toolbox tool WebFetch runs. Picked by measurement (2026-10-19) against + * context-dev/web-scrape-markdown, octen/extract, mrscraper/scrape-markdown, + * exa/contents, keenable/v1-fetch, surf/web-fetch and + * dataforseo/onpage-content-parsing on a plain article, a JavaScript-rendered + * page and a large Wikipedia article: + * - the lowest fixed price, $0.0013 per run (context-dev $0.003293, keenable + * $0.0052); octen and mrscraper are priced only after the run; + * - main-content Markdown that keeps links, like Firecrawl's scrape. It + * rendered the JavaScript-only page, and on the large page it fit more of the + * article into WebFetch's 100k-character window than the other Markdown + * tools did; + * - a 404 or an unresolvable host comes back as an unbilled `is_error` run, + * where exa and octen report success with an empty result. + * It was slower than context-dev (3.9-10.8 s against 2.4-10.2 s). + */ +export const AIMLAPI_WEB_FETCH_TOOL_ID = 'search1api/crawl' + +/** + * Spending cap sent with every run. The tool costs $0.0013 per run; the cap + * leaves room for a small price change, and a run priced above it is refused + * before it starts (400 max_cost_exceeded) instead of being charged. + */ +export const AIMLAPI_WEB_FETCH_MAX_COST_USD = 0.005 + +// How long POST /run may hold the request open before answering 202. +const RUN_WAIT_MS = 30_000 +// Upper bound for the whole scrape, polling included. +const DEFAULT_TIMEOUT_MS = 120_000 +const DEFAULT_MAX_ATTEMPTS = 3 +const DEFAULT_BACKOFF_MS = 500 +const MAX_RETRY_DELAY_MS = 10_000 +const DEFAULT_POLL_INTERVAL_MS = 1_000 +const MIN_POLL_INTERVAL_MS = 250 +const MAX_POLL_INTERVAL_MS = 5_000 +const TERMINAL_RUN_STATUSES = new Set(['completed', 'failed', 'stopped', 'timed_out']) +// Rate limits and gateway errors. Retrying them reuses the idempotency key, so +// a run that did start is returned again instead of being charged twice. +const RETRYABLE_HTTP_STATUSES = new Set([429, 502, 503, 504]) + +interface ToolboxRun { + id?: string + status?: string + is_error?: boolean + error?: { message?: string; kind?: string; retryable?: boolean } + output?: unknown + cost?: { quoted_usd?: number; actual_usd?: number; over_max?: boolean } + next_poll_hint_ms?: number +} + +interface ToolboxErrorBody { + error?: { message?: string; code?: string } +} + +interface AimlapiScrapeOptions { + apiKey?: string | null + signal?: AbortSignal + timeoutMs?: number + maxAttempts?: number + backoffMs?: number +} + +/** True when the user explicitly chose the AI/ML API backend for WebFetch. */ +export function isAimlapiWebFetchSelected(): boolean { + return process.env.WEB_FETCH_PROVIDER?.trim().toLowerCase() === 'aimlapi' +} + +/** + * An aimlapi provider profile pointed at the user's own proxy exports the + * proxy's credential as AIMLAPI_API_KEY. That key belongs to the proxy, so it + * must never be sent to tools.aimlapi.com: treat it as no key at all. + */ +function isProxiedAimlapiProfileKey(): boolean { + const baseUrl = process.env.OPENAI_BASE_URL?.trim() || process.env.OPENAI_API_BASE?.trim() + return ( + process.env.CLAUDE_CODE_PROVIDER_ROUTE_ID === 'aimlapi' && + Boolean(baseUrl) && + !isCanonicalAimlapiInferenceBaseUrl(baseUrl!) + ) +} + +/** + * A provider profile can export a comma-separated credential pool as + * AIMLAPI_API_KEY. Use its first usable key, as the web search backend does; + * there is no rotation or retry across keys for billed runs. + */ +function resolveApiKey(options: AimlapiScrapeOptions): string { + const ambientKey = isProxiedAimlapiProfileKey() + ? undefined + : firstUsableCredential(process.env.AIMLAPI_API_KEY) + const apiKey = (options.apiKey ?? ambientKey ?? '').trim() + if (!apiKey) { + throw new Error( + 'WEB_FETCH_PROVIDER=aimlapi needs an AI/ML API key. Set AIMLAPI_API_KEY, or unset WEB_FETCH_PROVIDER to fetch pages directly.', + ) + } + return apiKey +} + +function buildHeaders(apiKey: string): Record { + return { + Accept: 'application/json', + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + [SOURCE_HEADER_NAME]: AIMLAPI_SOURCE, + [PARTNER_HEADER_NAME]: resolvePartnerId(), + } +} + +async function readJson(response: Response): Promise { + const text = await response.text() + if (!text) return undefined + try { + return JSON.parse(text) as T + } catch { + return undefined + } +} + +function retryDelayMs(response: Response, attempt: number, backoffMs: number): number { + const retryAfterSeconds = Number(response.headers.get('retry-after')) + const delay = + Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0 + ? retryAfterSeconds * 1000 + : backoffMs * Math.pow(2, attempt) + return Math.min(delay, MAX_RETRY_DELAY_MS) +} + +function pollDelayMs(run: ToolboxRun): number { + const hint = run.next_poll_hint_ms + if (typeof hint !== 'number' || !Number.isFinite(hint)) return DEFAULT_POLL_INTERVAL_MS + return Math.min(Math.max(hint, MIN_POLL_INTERVAL_MS), MAX_POLL_INTERVAL_MS) +} + +/** Remove an echoed credential before error text is shown. */ +function redact(value: string | undefined, apiKey: string): string | undefined { + return value?.split(apiKey).join('[redacted]') +} + +async function requestError(response: Response, action: string, apiKey: string): Promise { + const body = await readJson(response) + const code = body?.error?.code ? ` (${redact(body.error.code, apiKey)})` : '' + const message = body?.error?.message ? `: ${redact(body.error.message, apiKey)}` : '' + return new Error(`AI/ML API Toolbox ${action} error ${response.status}${code}${message}`) +} + +async function startRun( + url: string, + apiKey: string, + headers: Record, + signal: AbortSignal, + maxAttempts: number, + backoffMs: number, +): Promise { + const body = JSON.stringify({ + tool: AIMLAPI_WEB_FETCH_TOOL_ID, + input: { url }, + max_cost_usd: AIMLAPI_WEB_FETCH_MAX_COST_USD, + wait_ms: RUN_WAIT_MS, + idempotency_key: randomUUID(), + }) + + for (let attempt = 0; ; attempt++) { + const response = await fetch(`${AIMLAPI_TOOLBOX_BASE_URL}/run`, { + method: 'POST', + headers, + body, + // Never follow a redirect: the key and attribution headers must not + // reach any host but the Toolbox. + redirect: 'error', + signal, + }) + if (response.ok) { + const run = await readJson(response) + if (!run) throw new Error('AI/ML API Toolbox run returned an invalid response') + return run + } + if (!RETRYABLE_HTTP_STATUSES.has(response.status) || attempt >= maxAttempts - 1) { + throw await requestError(response, 'run', apiKey) + } + await response.body?.cancel() + await sleep(retryDelayMs(response, attempt, backoffMs), signal) + } +} + +async function waitForRun( + run: ToolboxRun, + apiKey: string, + headers: Record, + signal: AbortSignal, +): Promise { + let current = run + while (!TERMINAL_RUN_STATUSES.has(current.status ?? '')) { + if (!current.id) throw new Error('AI/ML API Toolbox run is still pending but has no run id') + await sleep(pollDelayMs(current), signal) + const response = await fetch( + `${AIMLAPI_TOOLBOX_BASE_URL}/runs/${encodeURIComponent(current.id)}`, + { method: 'GET', headers, redirect: 'error', signal }, + ) + if (RETRYABLE_HTTP_STATUSES.has(response.status)) { + await response.body?.cancel() + continue + } + if (!response.ok) throw await requestError(response, 'run status', apiKey) + const next = await readJson(response) + if (!next) throw new Error('AI/ML API Toolbox run status returned an invalid response') + current = { ...next, id: next.id ?? current.id } + } + return current +} + +function extractMarkdown(output: unknown): string | undefined { + if (typeof output !== 'object' || output === null) return undefined + const results = (output as { results?: unknown }).results + if (typeof results !== 'object' || results === null) return undefined + const content = (results as { content?: unknown }).content + return typeof content === 'string' ? content : undefined +} + +function capNote(run: ToolboxRun): string { + return run.cost?.over_max + ? ` The run hit the $${AIMLAPI_WEB_FETCH_MAX_COST_USD} max_cost_usd cap.` + : '' +} + +/** + * Scrape a page to Markdown with the AI/ML API Toolbox. Throws on any failure, + * like the Firecrawl scrape: WebFetch reports the error rather than silently + * fetching the page another way. + */ +export async function scrapeWithAimlapi( + url: string, + options: AimlapiScrapeOptions = {}, +): Promise<{ markdown: string; bytes: number }> { + if (!url.trim()) { + throw new Error('AI/ML API Toolbox URL cannot be empty') + } + const apiKey = resolveApiKey(options) + const headers = buildHeaders(apiKey) + const { signal, cleanup } = createCombinedAbortSignal(options.signal, { + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }) + + try { + const started = await startRun( + url, + apiKey, + headers, + signal, + options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS, + options.backoffMs ?? DEFAULT_BACKOFF_MS, + ) + const run = await waitForRun(started, apiKey, headers, signal) + + if (run.is_error || run.status !== 'completed') { + const reason = redact(run.error?.message, apiKey) ?? `run ended with status ${redact(run.status, apiKey)}` + const retry = run.error?.retryable ? ' (retryable)' : '' + throw new Error( + `AI/ML API Toolbox ${AIMLAPI_WEB_FETCH_TOOL_ID} failed for ${url}: ${reason}${retry}.${capNote(run)}`, + ) + } + + const markdown = extractMarkdown(run.output) + if (!markdown) { + throw new Error( + `AI/ML API Toolbox ${AIMLAPI_WEB_FETCH_TOOL_ID} returned no page content for ${url}.${capNote(run)}`, + ) + } + return { markdown, bytes: Buffer.byteLength(markdown) } + } finally { + cleanup() + } +} diff --git a/src/utils/envFile.test.ts b/src/utils/envFile.test.ts index f6d173fa7..ea8577b2a 100644 --- a/src/utils/envFile.test.ts +++ b/src/utils/envFile.test.ts @@ -15,6 +15,7 @@ import { PROVIDER_SELECTION_FLAGS } from './providerSelectionFlags.js' const TEST_ENV_KEYS = [ 'NODE_OPTIONS', + 'AIMLAPI_API_KEY', 'AZURE_OPENAI_API_VERSION', 'CLAUDE_CODE_USE_OPENAI', 'CODEX_AUTH_JSON_PATH', @@ -57,6 +58,7 @@ const TEST_ENV_KEYS = [ 'WEB_CUSTOM_ALLOW_PRIVATE', 'WEB_CUSTOM_MAX_BODY_KB', 'WEB_CUSTOM_TIMEOUT_SEC', + 'WEB_FETCH_PROVIDER', 'WEB_HEADERS', 'WEB_JSON_PATH', 'WEB_KEY', @@ -383,6 +385,21 @@ describe('loadEnvFile', () => { expect(process.env.MERGE_GATEWAY_API_KEY).toBe('shell-merge-key') }) + it('loads the documented AI/ML API key and WebFetch provider values', () => { + const filePath = writeTempEnvFile( + ['AIMLAPI_API_KEY=aimlapi-key', 'WEB_FETCH_PROVIDER=aimlapi'].join('\n'), + ) + + const loaded = loadEnvFile(filePath) + + expect(process.env.AIMLAPI_API_KEY).toBe('aimlapi-key') + expect(process.env.WEB_FETCH_PROVIDER).toBe('aimlapi') + expect(loaded).toEqual({ + AIMLAPI_API_KEY: 'aimlapi-key', + WEB_FETCH_PROVIDER: 'aimlapi', + }) + }) + it('loads the dedicated LLMTR credential without selecting a route', () => { const filePath = writeTempEnvFile('LLMTR_API_KEY=llmtr-key') diff --git a/src/utils/envFile.ts b/src/utils/envFile.ts index 824eba5a5..adcfff4ab 100644 --- a/src/utils/envFile.ts +++ b/src/utils/envFile.ts @@ -3,6 +3,7 @@ import { readFileSync } from 'node:fs' const ENV_KEY_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/ const ALLOWED_ENV_FILE_KEYS = new Set([ + 'AIMLAPI_API_KEY', 'ANTHROPIC_API_KEY', 'ANTHROPIC_AUTH_TOKEN', 'ANTHROPIC_BASE_URL', @@ -140,6 +141,7 @@ const ALLOWED_ENV_FILE_KEYS = new Set([ 'WEB_CUSTOM_ALLOW_PRIVATE', 'WEB_CUSTOM_MAX_BODY_KB', 'WEB_CUSTOM_TIMEOUT_SEC', + 'WEB_FETCH_PROVIDER', 'WEB_HEADERS', 'WEB_JSON_PATH', 'WEB_KEY',