diff --git a/src/services/api/claude.lifecycle.test.ts b/src/services/api/claude.lifecycle.test.ts index 9e24b0047..3408497c0 100644 --- a/src/services/api/claude.lifecycle.test.ts +++ b/src/services/api/claude.lifecycle.test.ts @@ -22,6 +22,7 @@ import { asSystemPrompt } from '../../utils/systemPromptType.js' import { executeNonStreamingRequest, type Options, + queryHaiku, queryModelWithStreaming, } from './claude.js' import { EMPTY_USAGE } from './emptyUsage.js' @@ -32,6 +33,8 @@ const envKeys = [ 'ANTHROPIC_API_KEY', 'ANTHROPIC_BASE_URL', 'ANTHROPIC_MODEL', + 'ANTHROPIC_SMALL_FAST_MODEL', + 'CLAUDE_CODE_ALWAYS_ENABLE_EFFORT', 'CLAUDE_CODE_TEST_FIXTURES_ROOT', 'CLAUDE_CODE_PROVIDER_PROFILE_ENV_APPLIED', 'CLAUDE_CODE_PROVIDER_PROFILE_ENV_APPLIED_ID', @@ -166,6 +169,50 @@ function makeOpenAIStreamingResponse(): Response { ) } +function makeAnthropicStreamingResponse(): Response { + const events = [ + { + event: 'message_start', + data: { type: 'message_start', message: makeBetaMessage() }, + }, + { + event: 'content_block_start', + data: { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: '' }, + }, + }, + { + event: 'content_block_delta', + data: { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'ok' }, + }, + }, + { + event: 'content_block_stop', + data: { type: 'content_block_stop', index: 0 }, + }, + { + event: 'message_delta', + data: { + type: 'message_delta', + delta: { stop_reason: 'end_turn', stop_sequence: null }, + usage: { output_tokens: 1 }, + }, + }, + { event: 'message_stop', data: { type: 'message_stop' } }, + ] + const body = events + .map(event => `event: ${event.event}\ndata: ${JSON.stringify(event.data)}\n\n`) + .join('') + return new Response(body, { + headers: { 'content-type': 'text/event-stream' }, + }) +} + function makeStallingOpenAIStreamResponse( onCancel?: (reason: unknown) => void, ): Response { @@ -362,6 +409,43 @@ afterEach(() => { }) describe('Claude API lifecycle tracking', () => { + test('Haiku side queries omit effort from native Anthropic requests when force enabled', async () => { + setClientTestEnv() + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + process.env.ANTHROPIC_SMALL_FAST_MODEL = 'claude-haiku-4-5-20251001' + process.env.OPENCLAUDE_MAX_RETRIES = '0' + const queryLifecycle = new QueryLifecycleOperationTracker() + let requestBody: Record | undefined + let requestHeaders: Headers | undefined + const fetchOverride: FetchOverride = async (_input, init) => { + requestBody = parseRequestBody(init) + requestHeaders = new Headers(init?.headers) + return makeAnthropicStreamingResponse() + } + + await queryHaiku({ + userPrompt: 'summarize this turn', + systemPrompt: asSystemPrompt([]), + signal: new AbortController().signal, + options: { + isNonInteractiveSession: false, + querySource: 'sdk', + agents: [], + hasAppendSystemPrompt: false, + mcpTools: [], + queryLifecycle, + fetchOverride, + effortValue: 'medium', + }, + }) + + expect(requestBody?.model).toBe('claude-haiku-4-5-20251001') + expect(requestBody).not.toHaveProperty('output_config.effort') + expect(requestBody).not.toHaveProperty('reasoning_effort') + expect(requestBody).not.toHaveProperty('effort') + expect(requestHeaders?.get('anthropic-beta')).not.toContain('effort') + }) + test('uses the original codexplan selection for custom-gateway defaults', async () => { setClientTestEnv() process.env.CLAUDE_CODE_USE_OPENAI = '1' diff --git a/src/services/api/client.test.ts b/src/services/api/client.test.ts index 42237c65b..13696df80 100644 --- a/src/services/api/client.test.ts +++ b/src/services/api/client.test.ts @@ -22,6 +22,21 @@ const realProviders = { } mock.module('../../utils/model/providers.js', () => realProviders) mock.module('src/utils/model/providers.js', () => realProviders) +const _realModelSupportOverridesModule = await import( + `../../utils/model/modelSupportOverrides.js?real=${Date.now()}-${Math.random()}`, +) +const realModelSupportOverrides = { + get3PModelCapabilityOverride: + _realModelSupportOverridesModule.get3PModelCapabilityOverride, +} +mock.module( + '../../utils/model/modelSupportOverrides.js', + () => realModelSupportOverrides, +) +mock.module( + 'src/utils/model/modelSupportOverrides.js', + () => realModelSupportOverrides, +) const { getAnthropicClient } = await import( `./client.js?real=${Date.now()}-${Math.random()}`, ) @@ -39,6 +54,8 @@ type ShimClient = { const originalFetch = globalThis.fetch const originalMacro = (globalThis as Record).MACRO const originalEnv = { + CLAUDE_CODE_ALWAYS_ENABLE_EFFORT: + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT, CLAUDE_CODE_USE_OPENAI: process.env.CLAUDE_CODE_USE_OPENAI, CLAUDE_CODE_USE_BEDROCK: process.env.CLAUDE_CODE_USE_BEDROCK, CLAUDE_CODE_SKIP_BEDROCK_AUTH: process.env.CLAUDE_CODE_SKIP_BEDROCK_AUTH, @@ -79,6 +96,18 @@ const originalEnv = { ANTHROPIC_BASE_URL: process.env.ANTHROPIC_BASE_URL, ANTHROPIC_MODEL: process.env.ANTHROPIC_MODEL, ANTHROPIC_CUSTOM_HEADERS: process.env.ANTHROPIC_CUSTOM_HEADERS, + ANTHROPIC_DEFAULT_OPUS_MODEL: + process.env.ANTHROPIC_DEFAULT_OPUS_MODEL, + ANTHROPIC_DEFAULT_OPUS_MODEL_SUPPORTED_CAPABILITIES: + process.env.ANTHROPIC_DEFAULT_OPUS_MODEL_SUPPORTED_CAPABILITIES, + ANTHROPIC_DEFAULT_SONNET_MODEL: + process.env.ANTHROPIC_DEFAULT_SONNET_MODEL, + ANTHROPIC_DEFAULT_SONNET_MODEL_SUPPORTED_CAPABILITIES: + process.env.ANTHROPIC_DEFAULT_SONNET_MODEL_SUPPORTED_CAPABILITIES, + ANTHROPIC_DEFAULT_HAIKU_MODEL: + process.env.ANTHROPIC_DEFAULT_HAIKU_MODEL, + ANTHROPIC_DEFAULT_HAIKU_MODEL_SUPPORTED_CAPABILITIES: + process.env.ANTHROPIC_DEFAULT_HAIKU_MODEL_SUPPORTED_CAPABILITIES, USER_TYPE: process.env.USER_TYPE, USE_STAGING_OAUTH: process.env.USE_STAGING_OAUTH, CLAUDE_CODE_PROVIDER_PROFILE_ENV_APPLIED: @@ -133,6 +162,12 @@ function clearEnvForMiniMaxOnlyTest(): void { delete process.env.ANTHROPIC_BASE_URL delete process.env.ANTHROPIC_MODEL delete process.env.ANTHROPIC_CUSTOM_HEADERS + delete process.env.ANTHROPIC_DEFAULT_OPUS_MODEL + delete process.env.ANTHROPIC_DEFAULT_OPUS_MODEL_SUPPORTED_CAPABILITIES + delete process.env.ANTHROPIC_DEFAULT_SONNET_MODEL + delete process.env.ANTHROPIC_DEFAULT_SONNET_MODEL_SUPPORTED_CAPABILITIES + delete process.env.ANTHROPIC_DEFAULT_HAIKU_MODEL + delete process.env.ANTHROPIC_DEFAULT_HAIKU_MODEL_SUPPORTED_CAPABILITIES delete process.env.USER_TYPE delete process.env.USE_STAGING_OAUTH } @@ -146,6 +181,7 @@ beforeEach(async () => { process.env.GEMINI_BASE_URL = 'https://gemini.example/v1beta/openai' process.env.GEMINI_AUTH_MODE = 'api-key' + delete process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT delete process.env.CLAUDE_CODE_USE_OPENAI delete process.env.CLAUDE_CODE_USE_BEDROCK delete process.env.CLAUDE_CODE_SKIP_BEDROCK_AUTH @@ -181,6 +217,12 @@ beforeEach(async () => { delete process.env.ANTHROPIC_BASE_URL delete process.env.ANTHROPIC_MODEL delete process.env.ANTHROPIC_CUSTOM_HEADERS + delete process.env.ANTHROPIC_DEFAULT_OPUS_MODEL + delete process.env.ANTHROPIC_DEFAULT_OPUS_MODEL_SUPPORTED_CAPABILITIES + delete process.env.ANTHROPIC_DEFAULT_SONNET_MODEL + delete process.env.ANTHROPIC_DEFAULT_SONNET_MODEL_SUPPORTED_CAPABILITIES + delete process.env.ANTHROPIC_DEFAULT_HAIKU_MODEL + delete process.env.ANTHROPIC_DEFAULT_HAIKU_MODEL_SUPPORTED_CAPABILITIES delete process.env.CLAUDE_CODE_PROVIDER_PROFILE_ENV_APPLIED delete process.env.CLAUDE_CODE_PROVIDER_PROFILE_ENV_APPLIED_ID }) @@ -188,6 +230,10 @@ beforeEach(async () => { afterEach(() => { try { ;(globalThis as Record).MACRO = originalMacro + restoreEnv( + 'CLAUDE_CODE_ALWAYS_ENABLE_EFFORT', + originalEnv.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT, + ) restoreEnv('CLAUDE_CODE_USE_OPENAI', originalEnv.CLAUDE_CODE_USE_OPENAI) restoreEnv('CLAUDE_CODE_USE_BEDROCK', originalEnv.CLAUDE_CODE_USE_BEDROCK) restoreEnv( @@ -231,6 +277,30 @@ afterEach(() => { restoreEnv('ANTHROPIC_BASE_URL', originalEnv.ANTHROPIC_BASE_URL) restoreEnv('ANTHROPIC_MODEL', originalEnv.ANTHROPIC_MODEL) restoreEnv('ANTHROPIC_CUSTOM_HEADERS', originalEnv.ANTHROPIC_CUSTOM_HEADERS) + restoreEnv( + 'ANTHROPIC_DEFAULT_OPUS_MODEL', + originalEnv.ANTHROPIC_DEFAULT_OPUS_MODEL, + ) + restoreEnv( + 'ANTHROPIC_DEFAULT_OPUS_MODEL_SUPPORTED_CAPABILITIES', + originalEnv.ANTHROPIC_DEFAULT_OPUS_MODEL_SUPPORTED_CAPABILITIES, + ) + restoreEnv( + 'ANTHROPIC_DEFAULT_SONNET_MODEL', + originalEnv.ANTHROPIC_DEFAULT_SONNET_MODEL, + ) + restoreEnv( + 'ANTHROPIC_DEFAULT_SONNET_MODEL_SUPPORTED_CAPABILITIES', + originalEnv.ANTHROPIC_DEFAULT_SONNET_MODEL_SUPPORTED_CAPABILITIES, + ) + restoreEnv( + 'ANTHROPIC_DEFAULT_HAIKU_MODEL', + originalEnv.ANTHROPIC_DEFAULT_HAIKU_MODEL, + ) + restoreEnv( + 'ANTHROPIC_DEFAULT_HAIKU_MODEL_SUPPORTED_CAPABILITIES', + originalEnv.ANTHROPIC_DEFAULT_HAIKU_MODEL_SUPPORTED_CAPABILITIES, + ) restoreEnv('USER_TYPE', originalEnv.USER_TYPE) restoreEnv('USE_STAGING_OAUTH', originalEnv.USE_STAGING_OAUTH) restoreEnv( @@ -1854,9 +1924,9 @@ test('auto-routed Azure gpt-5.4 and gpt-5.5 requests preserve selected effort', ]) }) -test('OPENAI_API_BASE gateway does not inherit first-party GPT-5.6 effort metadata', async () => { +test('OPENAI_API_BASE gateway omits the GPT-5.6 default but preserves explicit effort', async () => { let requestUrl = '' - let requestBody: Record | undefined + const requestBodies: Record[] = [] delete process.env.CLAUDE_CODE_USE_GEMINI delete process.env.GEMINI_API_KEY process.env.CLAUDE_CODE_USE_OPENAI = '1' @@ -1865,7 +1935,7 @@ test('OPENAI_API_BASE gateway does not inherit first-party GPT-5.6 effort metada globalThis.fetch = (async (input, init) => { requestUrl = String(input) - requestBody = JSON.parse(String(init?.body)) + requestBodies.push(JSON.parse(String(init?.body))) return new Response(JSON.stringify({ id: 'chatcmpl-gateway', choices: [{ message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }], @@ -1873,15 +1943,22 @@ test('OPENAI_API_BASE gateway does not inherit first-party GPT-5.6 effort metada }), { headers: { 'Content-Type': 'application/json' } }) }) as FetchType - const client = (await getAnthropicClient({ + const defaultClient = (await getAnthropicClient({ + maxRetries: 0, + model: 'gpt-5.6-sol', + })) as unknown as ShimClient + await defaultClient.beta.messages.create({ model: 'gpt-5.6-sol', messages: [{ role: 'user', content: 'hello' }], max_tokens: 64, stream: false }) + + const explicitClient = (await getAnthropicClient({ maxRetries: 0, model: 'gpt-5.6-sol', effortValue: 'xhigh', })) as unknown as ShimClient - await client.beta.messages.create({ model: 'gpt-5.6-sol', messages: [{ role: 'user', content: 'hello' }], max_tokens: 64, stream: false }) + await explicitClient.beta.messages.create({ model: 'gpt-5.6-sol', messages: [{ role: 'user', content: 'hello' }], max_tokens: 64, stream: false }) expect(requestUrl).toBe('https://gateway.example/v1/chat/completions') - expect(requestBody?.reasoning_effort).toBeUndefined() + expect(requestBodies[0]?.reasoning_effort).toBeUndefined() + expect(requestBodies[1]?.reasoning_effort).toBe('xhigh') }) test('providerOverride Azure gpt effort uses the override base for catalog metadata', async () => { @@ -2009,7 +2086,7 @@ test('providerOverride custom OpenAI-compatible gpt effort uses legacy support', const client = (await getAnthropicClient({ maxRetries: 0, - effortValue: 'high', + effortValue: 'medium', providerOverride: { model: 'gpt-5.4', baseURL: 'https://custom-openai-compatible.example.test/v1', @@ -2025,8 +2102,372 @@ test('providerOverride custom OpenAI-compatible gpt effort uses legacy support', stream: false, }) - expect(requestBody?.reasoning_effort).toBe('high') + expect(requestBody?.reasoning_effort).toBe('medium') }) + +test('providerOverride custom shims do not infer native Claude or Gemini effort', async () => { + const requestBodies: Record[] = [] + + globalThis.fetch = (async (_input, init) => { + const requestBody = JSON.parse(String(init?.body)) + requestBodies.push(requestBody) + return new Response( + JSON.stringify({ + id: 'chatcmpl-provider-override-native-name', + model: requestBody.model, + choices: [ + { + message: { role: 'assistant', content: 'ok' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 8, + completion_tokens: 3, + total_tokens: 11, + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ) + }) as FetchType + + for (const model of ['claude-opus-4-5', 'gemini-3-pro']) { + const client = (await getAnthropicClient({ + maxRetries: 0, + effortValue: 'medium', + providerOverride: { + model, + baseURL: 'https://custom-openai-compatible.example.test/v1', + apiKey: 'provider-test-key', + }, + })) as unknown as ShimClient + + await client.beta.messages.create({ + model: 'unused', + messages: [{ role: 'user', content: 'hello' }], + max_tokens: 64, + stream: false, + }) + } + + expect(requestBodies.map(body => body.model)).toEqual([ + 'claude-opus-4-5', + 'gemini-3-pro', + ]) + for (const requestBody of requestBodies) { + expect(requestBody.reasoning_effort).toBeUndefined() + } + + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + const forceEnabledGemini = (await getAnthropicClient({ + maxRetries: 0, + effortValue: 'medium', + providerOverride: { + model: 'gemini-3-pro', + baseURL: 'https://custom-openai-compatible.example.test/v1', + apiKey: 'provider-test-key', + }, + })) as unknown as ShimClient + await forceEnabledGemini.beta.messages.create({ + model: 'unused', + messages: [{ role: 'user', content: 'hello' }], + max_tokens: 64, + stream: false, + }) + expect(requestBodies).toHaveLength(3) + expect(requestBodies[2]!.reasoning_effort).toBeUndefined() +}) + +test('NVIDIA NIM does not infer native Claude effort from the model name', async () => { + let requestBody: Record | undefined + delete process.env.CLAUDE_CODE_USE_GEMINI + delete process.env.GEMINI_API_KEY + process.env.NVIDIA_NIM = '1' + process.env.NVIDIA_API_KEY = 'nvidia-test-key' + process.env.OPENAI_BASE_URL = 'https://integrate.api.nvidia.com/v1' + process.env.OPENAI_MODEL = 'claude-opus-4-5' + + globalThis.fetch = (async (_input, init) => { + requestBody = JSON.parse(String(init?.body)) + return new Response( + JSON.stringify({ + id: 'chatcmpl-nvidia-native-name', + model: 'claude-opus-4-5', + choices: [ + { + message: { role: 'assistant', content: 'ok' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 8, + completion_tokens: 3, + total_tokens: 11, + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ) + }) as FetchType + + const client = (await getAnthropicClient({ + maxRetries: 0, + model: 'claude-opus-4-5', + effortValue: 'medium', + })) as unknown as ShimClient + await client.beta.messages.create({ + model: 'claude-opus-4-5', + messages: [{ role: 'user', content: 'hello' }], + max_tokens: 64, + stream: false, + }) + + expect(requestBody?.reasoning_effort).toBeUndefined() +}) + +test('OpenCode native endpoints preserve authorized Claude and Gemini effort', async () => { + const requests: Array<{ + url: string + body: Record + }> = [] + delete process.env.CLAUDE_CODE_USE_GEMINI + delete process.env.GEMINI_API_KEY + process.env.CLAUDE_CODE_USE_OPENAI = '1' + process.env.OPENAI_API_KEY = 'opencode-test-key' + process.env.OPENAI_BASE_URL = 'https://opencode.ai/zen/v1' + + globalThis.fetch = (async (input, init) => { + const url = String(input) + requests.push({ url, body: JSON.parse(String(init?.body)) }) + if (url.includes('/messages')) { + return new Response( + JSON.stringify({ + id: 'msg_opencode_effort', + type: 'message', + role: 'assistant', + model: 'claude-opus-4-5', + content: [{ type: 'text', text: 'ok' }], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 8, output_tokens: 3 }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ) + } + return new Response( + JSON.stringify({ + candidates: [ + { + content: { role: 'model', parts: [{ text: 'ok' }] }, + finishReason: 'STOP', + }, + ], + usageMetadata: { + promptTokenCount: 8, + candidatesTokenCount: 3, + totalTokenCount: 11, + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ) + }) as FetchType + + for (const model of ['claude-opus-4-5', 'gemini-3.1-pro']) { + process.env.OPENAI_MODEL = model + const client = (await getAnthropicClient({ + maxRetries: 0, + model, + effortValue: 'medium', + })) as unknown as ShimClient + await client.beta.messages.create({ + model, + messages: [{ role: 'user', content: 'hello' }], + max_tokens: 64, + stream: false, + }) + } + + expect(requests[0]?.url).toEndWith('/messages') + expect(requests[0]?.body.effort).toBe('medium') + expect(requests[1]?.url).toContain('/models/gemini-3.1-pro') + expect(requests[1]?.body.generationConfig).toMatchObject({ + thinkingConfig: { + includeThoughts: true, + thinkingLevel: 'medium', + }, + }) +}) + +test('force enable adds effort only for an otherwise unresolved custom shim model', async () => { + const requestBodies: Record[] = [] + + globalThis.fetch = (async (_input, init) => { + requestBodies.push(JSON.parse(String(init?.body))) + return new Response( + JSON.stringify({ + id: 'chatcmpl-provider-override-force-effort', + model: 'gateway-custom-model', + choices: [ + { + message: { role: 'assistant', content: 'ok' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 8, + completion_tokens: 3, + total_tokens: 11, + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ) + }) as FetchType + + for (const forceEnabled of [false, true]) { + if (forceEnabled) { + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + } else { + delete process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT + } + const client = (await getAnthropicClient({ + maxRetries: 0, + effortValue: 'medium', + providerOverride: { + model: 'gateway-custom-model', + baseURL: 'https://custom-openai-compatible.example.test/v1', + apiKey: 'provider-test-key', + }, + })) as unknown as ShimClient + + await client.beta.messages.create({ + model: 'unused', + messages: [{ role: 'user', content: 'hello' }], + max_tokens: 64, + stream: false, + }) + } + + expect(requestBodies[0]?.reasoning_effort).toBeUndefined() + expect(requestBodies[1]?.reasoning_effort).toBe('medium') +}) + +test('providerOverride third-party false beats force enable under a first-party parent', async () => { + let requestBody: Record | undefined + delete process.env.CLAUDE_CODE_USE_GEMINI + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + process.env.ANTHROPIC_DEFAULT_SONNET_MODEL = 'gateway-custom-model' + process.env.ANTHROPIC_DEFAULT_SONNET_MODEL_SUPPORTED_CAPABILITIES = '' + + globalThis.fetch = (async (_input, init) => { + requestBody = JSON.parse(String(init?.body)) + return new Response( + JSON.stringify({ + id: 'chatcmpl-provider-override-force-false', + model: 'gateway-custom-model', + choices: [ + { + message: { role: 'assistant', content: 'ok' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 8, + completion_tokens: 3, + total_tokens: 11, + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ) + }) as FetchType + + const client = (await getAnthropicClient({ + maxRetries: 0, + effortValue: 'medium', + providerOverride: { + model: 'gateway-custom-model', + baseURL: 'https://custom-openai-compatible.example.test/v1', + apiKey: 'provider-test-key', + }, + })) as unknown as ShimClient + + await client.beta.messages.create({ + model: 'unused', + messages: [{ role: 'user', content: 'hello' }], + max_tokens: 64, + stream: false, + }) + + expect(requestBody?.reasoning_effort).toBeUndefined() +}) + +test('force enable respects a shim non-effort contract before request serialization', async () => { + let requestBody: Record | undefined + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + + globalThis.fetch = (async (_input, init) => { + requestBody = JSON.parse(String(init?.body)) + return new Response( + JSON.stringify({ + id: 'chatcmpl-provider-override-force-veto', + model: 'gateway-custom-model', + choices: [ + { + message: { role: 'assistant', content: 'ok' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 8, + completion_tokens: 3, + total_tokens: 11, + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ) + }) as FetchType + + _clearRegistryForTesting() + try { + registerGateway({ + id: 'non-effort-transport-test', + label: 'Non-effort Transport Test', + defaultBaseUrl: 'https://non-effort-transport.example.test/v1', + setup: { requiresAuth: true, authMode: 'api-key' }, + transportConfig: { + kind: 'openai-compatible', + openaiShim: { thinkingRequestFormat: 'none' }, + }, + catalog: { source: 'static', models: [] }, + }) + + const client = (await getAnthropicClient({ + maxRetries: 0, + effortValue: 'medium', + providerOverride: { + model: 'gateway-custom-model', + baseURL: 'https://non-effort-transport.example.test/v1', + apiKey: 'provider-test-key', + }, + })) as unknown as ShimClient + + // Remove the synthetic route before serialization so no later request-body + // cleanup can hide an incorrect reasoningEffort value captured by the client. + _clearRegistryForTesting() + ensureIntegrationsLoaded() + + await client.beta.messages.create({ + model: 'unused', + messages: [{ role: 'user', content: 'hello' }], + max_tokens: 64, + stream: false, + }) + } finally { + _clearRegistryForTesting() + ensureIntegrationsLoaded() + } + + expect(requestBody?.reasoning_effort).toBeUndefined() +}) + test('providerOverride clamps stale effort against metadata levels', async () => { let requestBody: Record | undefined diff --git a/src/services/api/client.ts b/src/services/api/client.ts index 42cb53065..ba3e585c3 100644 --- a/src/services/api/client.ts +++ b/src/services/api/client.ts @@ -504,11 +504,9 @@ export async function getAnthropicClient({ openaiShimConfig: effortShimConfig, baseUrl: effortBaseUrl, processEnv: effortProcessEnv, - apiProvider: effortRuntimeContext.routeId === 'openai' - ? 'openai' as const - : effortRuntimeContext.routeId === 'codex' - ? 'codex' as const - : undefined, + // Per-agent overrides always use the OpenAI-compatible shim, regardless + // of the parent session's provider. + apiProvider: providerOverride ? 'openai' as const : getAPIProvider(), } : undefined const supportsShimReasoningEffort = effortModel @@ -519,7 +517,7 @@ export async function getAnthropicClient({ effortShimConfig.removeBodyFields, effortContext, ) - : modelSupportsWireEffort(effortModel) + : modelSupportsWireEffort(effortModel, effortContext) : false const reasoningControl = effortModel ? resolveModelReasoningControl(effortModel, effortContext) diff --git a/src/utils/effort.codex.test.ts b/src/utils/effort.codex.test.ts index aa521eae9..c7a87d201 100644 --- a/src/utils/effort.codex.test.ts +++ b/src/utils/effort.codex.test.ts @@ -13,6 +13,35 @@ import * as actualAuth from './auth.js' import * as actualThinking from './thinking.js' import * as actualGrowthbook from 'src/services/analytics/growthbook.js' import * as actualModelSupportOverrides from './model/modelSupportOverrides.js' +import type { APIProvider } from './model/providers.js' + +type MockedThirdPartyCapability = 'effort' | 'max_effort' | 'xhigh_effort' + +const originalEnv = { ...process.env } +const routingEnvKeys = [ + 'ANTHROPIC_BASE_URL', + 'CLAUDE_CODE_ALWAYS_ENABLE_EFFORT', + 'CLAUDE_CODE_EFFORT_LEVEL', + 'CLAUDE_CODE_USE_BEDROCK', + 'CLAUDE_CODE_USE_FOUNDRY', + 'CLAUDE_CODE_USE_GEMINI', + 'CLAUDE_CODE_USE_GITHUB', + 'CLAUDE_CODE_USE_MISTRAL', + 'CLAUDE_CODE_USE_OPENAI', + 'CLAUDE_CODE_USE_VERTEX', + 'GEMINI_API_KEY', + 'MIMO_API_KEY', + 'MINIMAX_API_KEY', + 'NVIDIA_API_KEY', + 'NVIDIA_NIM', + 'OPENAI_API_BASE', + 'OPENAI_API_KEY', + 'OPENAI_BASE_URL', + 'OPENAI_MODEL', + 'XAI_API_KEY', + 'ZAI_API_KEY', + 'USER_TYPE', +] as const function restoreMockedModulesToActual(): void { mock.module('./model/modelSupportOverrides.js', () => actualModelSupportOverrides) @@ -21,31 +50,57 @@ function restoreMockedModulesToActual(): void { mock.module('src/services/analytics/growthbook.js', () => actualGrowthbook) } +function restoreProcessEnv(): void { + for (const key of Object.keys(process.env)) { + if (!Object.hasOwn(originalEnv, key)) delete process.env[key] + } + for (const [key, value] of Object.entries(originalEnv)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } +} beforeEach(async () => { await acquireSharedMutationLock('utils/effort.codex.test.ts') + for (const key of routingEnvKeys) { + delete process.env[key] + } }) afterEach(() => { try { mock.restore() restoreMockedModulesToActual() + restoreProcessEnv() } finally { releaseSharedMutationLock() } }) async function importFreshEffortModule(options: { - provider: 'codex' | 'openai' + provider: APIProvider supportsCodexReasoningEffort: boolean routeId?: string catalogEntries?: any[] modelDescriptors?: Record openaiShimConfig?: any + thirdPartyCapabilityOverrides?: { + apiProvider: APIProvider + capabilities: Partial> + } + useRuntimeFallback?: boolean }) { mock.module('./model/modelSupportOverrides.js', () => ({ ...actualModelSupportOverrides, - get3PModelCapabilityOverride: () => undefined, + get3PModelCapabilityOverride: ( + _model: string, + capability: MockedThirdPartyCapability, + apiProvider?: APIProvider, + ) => { + const override = options.thirdPartyCapabilityOverrides + if (!override || apiProvider !== override.apiProvider) return undefined + return override.capabilities[capability] + }, })) mock.module('./auth.js', () => ({ ...actualAuth, @@ -70,7 +125,8 @@ async function importFreshEffortModule(options: { options.routeId !== undefined || options.catalogEntries !== undefined || options.modelDescriptors !== undefined || - options.openaiShimConfig !== undefined + options.openaiShimConfig !== undefined || + options.useRuntimeFallback !== undefined ) ? { apiProvider: options.provider, @@ -79,6 +135,7 @@ async function importFreshEffortModule(options: { catalogEntries: options.catalogEntries, modelDescriptors: options.modelDescriptors, openaiShimConfig: options.openaiShimConfig, + useRuntimeFallback: options.useRuntimeFallback, } : undefined @@ -86,6 +143,17 @@ async function importFreshEffortModule(options: { ...effort, resolveModelReasoningControl: (model: string) => effort.resolveModelReasoningControl(model, reasoningContext), + resolveModelReasoningControlWithCompatibility: ( + model: string, + compatibilityOverrides: { + thinkingRequestFormat?: 'none' | 'deepseek-compatible' | 'zai-compatible' + removeBodyFields?: string[] + }, + ) => effort.resolveModelReasoningControl( + model, + reasoningContext, + compatibilityOverrides, + ), modelSupportsEffort: (model: string) => effort.modelSupportsEffort(model, reasoningContext), modelSupportsWireEffort: (model: string) => @@ -319,7 +387,7 @@ test('e2e: xhigh → persisted xhigh → resolveAppliedEffort → wire xhigh on test('e2e: max on non-Opus Anthropic model still clamps to high', async () => { const { resolveAppliedEffort } = await importFreshEffortModule({ - provider: 'firstParty' as unknown as 'openai', + provider: 'firstParty', supportsCodexReasoningEffort: false, }) @@ -328,7 +396,7 @@ test('e2e: max on non-Opus Anthropic model still clamps to high', async () => { test('modelSupportsXHighEffort: opus-4-7 and opus-4-8 are allowed; other Claude models are not', async () => { const { modelSupportsXHighEffort } = await importFreshEffortModule({ - provider: 'firstParty' as unknown as 'openai', + provider: 'firstParty', supportsCodexReasoningEffort: false, }) @@ -344,7 +412,7 @@ test('modelSupportsXHighEffort: opus-4-7 and opus-4-8 are allowed; other Claude test('xhigh does not appear in available levels for non-supporting models', async () => { const { getAvailableEffortLevels } = await importFreshEffortModule({ - provider: 'firstParty' as unknown as 'openai', + provider: 'firstParty', supportsCodexReasoningEffort: false, }) @@ -368,7 +436,7 @@ test('effort allowlist is narrowed to the shim isAdaptive||isOpus45 set', async // effort for them would silently drop low/medium on the wire. const { modelSupportsEffort, getAvailableEffortLevels } = await importFreshEffortModule({ - provider: 'firstParty' as unknown as 'openai', + provider: 'firstParty', supportsCodexReasoningEffort: false, }) @@ -398,7 +466,7 @@ test('effort allowlist is narrowed to the shim isAdaptive||isOpus45 set', async test('xhigh clamps to high on non-supporting models so stale settings.json values do not produce API errors', async () => { const { resolveAppliedEffort } = await importFreshEffortModule({ - provider: 'firstParty' as unknown as 'openai', + provider: 'firstParty', supportsCodexReasoningEffort: false, }) @@ -412,6 +480,9 @@ test('clampUltracodeEffort: clamps to xhigh on non-firstParty xhigh-capable mode const { clampUltracodeEffort, resolveAppliedEffort } = await importFreshEffortModule({ provider: 'openai', supportsCodexReasoningEffort: true, + routeId: 'opencode', + useRuntimeFallback: false, + openaiShimConfig: { endpointPath: '/messages' }, }) // ultracode isn't selectable off firstParty, so it clamps — but to xhigh @@ -428,7 +499,7 @@ test('clampUltracodeEffort: clamps to xhigh on non-firstParty xhigh-capable mode test('clampUltracodeEffort: clamps to high on firstParty non-xhigh model', async () => { const { clampUltracodeEffort, resolveAppliedEffort } = await importFreshEffortModule({ - provider: 'firstParty' as unknown as 'openai', + provider: 'firstParty', supportsCodexReasoningEffort: false, }) @@ -1014,6 +1085,253 @@ test('explicit non-controllable metadata opts out even when the model matches le expect(resolveAppliedEffort('gpt-5.4', 'high')).toBeUndefined() }) +test('force enable cannot override non-effort metadata or transport contracts', async () => { + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + const metadata = await importFreshEffortModule({ + provider: 'openai', + supportsCodexReasoningEffort: true, + routeId: 'custom-gateway', + useRuntimeFallback: false, + thirdPartyCapabilityOverrides: { + apiProvider: 'openai', + capabilities: { effort: true }, + }, + catalogEntries: [ + { + id: 'metadata-no-effort', + apiName: 'metadata-no-effort', + capabilities: { supportsReasoning: true }, + reasoning: { + mode: 'always-on', + wireFormat: 'none', + }, + }, + ], + }) + + expect(metadata.modelSupportsEffort('metadata-no-effort')).toBe(false) + expect(metadata.modelSupportsShimReasoningEffort('metadata-no-effort')).toBe(false) + expect(metadata.modelSupportsWireEffort('metadata-no-effort')).toBe(false) + expect(metadata.resolveAppliedEffort('metadata-no-effort', 'high')).toBeUndefined() + + const transport = await importFreshEffortModule({ + provider: 'openai', + supportsCodexReasoningEffort: true, + routeId: 'custom', + useRuntimeFallback: false, + thirdPartyCapabilityOverrides: { + apiProvider: 'openai', + capabilities: { effort: true }, + }, + openaiShimConfig: { thinkingRequestFormat: 'none' }, + }) + + expect(transport.modelSupportsEffort('transport-no-effort')).toBe(false) + expect(transport.modelSupportsShimReasoningEffort('transport-no-effort')).toBe(false) + expect(transport.modelSupportsWireEffort('transport-no-effort')).toBe(false) + expect(transport.resolveAppliedEffort('transport-no-effort', 'high')).toBeUndefined() + + const metadataWithTransportVeto = await importFreshEffortModule({ + provider: 'openai', + supportsCodexReasoningEffort: true, + routeId: 'custom-gateway', + useRuntimeFallback: false, + thirdPartyCapabilityOverrides: { + apiProvider: 'openai', + capabilities: { effort: true }, + }, + openaiShimConfig: { thinkingRequestFormat: 'none' }, + catalogEntries: [ + { + id: 'metadata-with-transport-veto', + apiName: 'metadata-with-transport-veto', + capabilities: { supportsReasoning: true }, + reasoning: { + mode: 'levels', + levels: ['low', 'medium', 'high'], + wireFormat: 'reasoning_effort', + }, + }, + ], + }) + + expect( + metadataWithTransportVeto.modelSupportsEffort( + 'metadata-with-transport-veto', + ), + ).toBe(false) + expect( + metadataWithTransportVeto.modelSupportsShimReasoningEffort( + 'metadata-with-transport-veto', + ), + ).toBe(false) + expect( + metadataWithTransportVeto.modelSupportsWireEffort( + 'metadata-with-transport-veto', + ), + ).toBe(false) +}) + +test('resolver and shim effort predicate accept explicit transport vetoes', async () => { + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + const { + modelSupportsShimReasoningEffort, + resolveModelReasoningControlWithCompatibility, + } = await importFreshEffortModule({ + provider: 'openai', + supportsCodexReasoningEffort: false, + routeId: 'custom', + useRuntimeFallback: false, + }) + + expect( + resolveModelReasoningControlWithCompatibility( + 'transport-no-effort', + { thinkingRequestFormat: 'none' }, + ), + ).toMatchObject({ + supportsReasoning: false, + controllable: false, + source: 'compat', + }) + + expect( + modelSupportsShimReasoningEffort('transport-no-effort', 'none'), + ).toBe(false) + expect( + modelSupportsShimReasoningEffort( + 'transport-no-effort', + undefined, + ['reasoning_effort'], + ), + ).toBe(false) +}) + +test('third-party false beats force enable for unresolved models', async () => { + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + const { + modelSupportsEffort, + modelSupportsShimReasoningEffort, + modelSupportsWireEffort, + resolveAppliedEffort, + } = await importFreshEffortModule({ + provider: 'openai', + supportsCodexReasoningEffort: false, + routeId: 'custom', + useRuntimeFallback: false, + thirdPartyCapabilityOverrides: { + apiProvider: 'openai', + capabilities: { effort: false }, + }, + }) + + expect(modelSupportsEffort('third-party-custom-model')).toBe(false) + expect(modelSupportsShimReasoningEffort('third-party-custom-model')).toBe(false) + expect(modelSupportsWireEffort('third-party-custom-model')).toBe(false) + expect(resolveAppliedEffort('third-party-custom-model', 'high')).toBeUndefined() +}) + +test('third-party effort overrides require the matching API provider', async () => { + const matchingProvider = await importFreshEffortModule({ + provider: 'bedrock', + supportsCodexReasoningEffort: false, + routeId: 'bedrock', + useRuntimeFallback: false, + thirdPartyCapabilityOverrides: { + apiProvider: 'bedrock', + capabilities: { + effort: true, + max_effort: true, + xhigh_effort: false, + }, + }, + }) + + expect( + matchingProvider.modelSupportsEffort('provider-scoped-model'), + ).toBe(true) + expect( + matchingProvider.getAvailableEffortLevels('provider-scoped-model'), + ).toEqual(['low', 'medium', 'high', 'max']) + + const xhighProvider = await importFreshEffortModule({ + provider: 'foundry', + supportsCodexReasoningEffort: false, + routeId: 'foundry', + useRuntimeFallback: false, + thirdPartyCapabilityOverrides: { + apiProvider: 'foundry', + capabilities: { + effort: true, + max_effort: false, + xhigh_effort: true, + }, + }, + }) + + expect( + xhighProvider.getAvailableEffortLevels('provider-scoped-model'), + ).toEqual(['low', 'medium', 'high', 'xhigh']) + + const differentProvider = await importFreshEffortModule({ + provider: 'vertex', + supportsCodexReasoningEffort: false, + routeId: 'vertex', + useRuntimeFallback: false, + thirdPartyCapabilityOverrides: { + apiProvider: 'bedrock', + capabilities: { + effort: true, + max_effort: true, + xhigh_effort: false, + }, + }, + }) + + expect( + differentProvider.modelSupportsEffort('provider-scoped-model'), + ).toBe(false) + expect( + differentProvider.getAvailableEffortLevels('provider-scoped-model'), + ).toEqual([]) +}) + +test('explicit effort metadata beats a third-party false override', async () => { + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + const { + modelSupportsEffort, + modelSupportsShimReasoningEffort, + modelSupportsWireEffort, + resolveAppliedEffort, + } = await importFreshEffortModule({ + provider: 'openai', + supportsCodexReasoningEffort: false, + routeId: 'custom-gateway', + useRuntimeFallback: false, + thirdPartyCapabilityOverrides: { + apiProvider: 'openai', + capabilities: { effort: false }, + }, + catalogEntries: [ + { + id: 'metadata-effort-model', + apiName: 'metadata-effort-model', + capabilities: { supportsReasoning: true }, + reasoning: { + mode: 'levels', + levels: ['low', 'medium', 'high'], + wireFormat: 'reasoning_effort', + }, + }, + ], + }) + + expect(modelSupportsEffort('metadata-effort-model')).toBe(true) + expect(modelSupportsShimReasoningEffort('metadata-effort-model')).toBe(true) + expect(modelSupportsWireEffort('metadata-effort-model')).toBe(true) + expect(resolveAppliedEffort('metadata-effort-model', 'high')).toBe('high') +}) + test('toggle reasoning metadata stays non-controllable until toggle serialization exists', async () => { const { getAvailableEffortLevels, @@ -1108,7 +1426,8 @@ test('compat DeepSeek routes stay non-controllable when the runtime shim strips expect(resolveModelReasoningControl('deepseek-r1-distill-llama-70b')).toMatchObject({ supportsReasoning: false, controllable: false, - source: 'none', + source: 'compat', + levels: [], }) expect(modelSupportsEffort('deepseek-r1-distill-llama-70b')).toBe(false) expect(modelSupportsWireEffort('deepseek-r1-distill-llama-70b')).toBe(false) diff --git a/src/utils/effort.test.ts b/src/utils/effort.test.ts index 4245b1ef4..ea7b9089d 100644 --- a/src/utils/effort.test.ts +++ b/src/utils/effort.test.ts @@ -3,17 +3,75 @@ import { acquireSharedMutationLock, releaseSharedMutationLock, } from '../test/sharedMutationLock.js' +import { + ensureIntegrationsLoaded, + getCatalogEntriesForRoute, +} from '../integrations/index.js' import * as realAuth from './auth.js' import * as realThinking from './thinking.js' +const realModelSupportOverridesModule = await import( + `./model/modelSupportOverrides.js?real=${Date.now()}-${Math.random()}`, +) +const realModelSupportOverrides = { + get3PModelCapabilityOverride: + realModelSupportOverridesModule.get3PModelCapabilityOverride, +} const originalEnv = { ...process.env } +const routingEnvKeys = [ + 'ANTHROPIC_BASE_URL', + 'ANTHROPIC_DEFAULT_HAIKU_MODEL', + 'ANTHROPIC_DEFAULT_HAIKU_MODEL_SUPPORTED_CAPABILITIES', + 'ANTHROPIC_DEFAULT_OPUS_MODEL', + 'ANTHROPIC_DEFAULT_OPUS_MODEL_SUPPORTED_CAPABILITIES', + 'ANTHROPIC_DEFAULT_SONNET_MODEL', + 'ANTHROPIC_DEFAULT_SONNET_MODEL_SUPPORTED_CAPABILITIES', + 'CLAUDE_CODE_ALWAYS_ENABLE_EFFORT', + 'CLAUDE_CODE_EFFORT_LEVEL', + 'CLAUDE_CODE_USE_BEDROCK', + 'CLAUDE_CODE_USE_FOUNDRY', + 'CLAUDE_CODE_USE_GEMINI', + 'CLAUDE_CODE_USE_GITHUB', + 'CLAUDE_CODE_USE_MISTRAL', + 'CLAUDE_CODE_USE_OPENAI', + 'CLAUDE_CODE_USE_VERTEX', + 'GEMINI_API_KEY', + 'MIMO_API_KEY', + 'MINIMAX_API_KEY', + 'NVIDIA_API_KEY', + 'NVIDIA_NIM', + 'OPENAI_API_BASE', + 'OPENAI_API_KEY', + 'OPENAI_BASE_URL', + 'OPENAI_MODEL', + 'XAI_API_KEY', + 'ZAI_API_KEY', + 'USER_TYPE', +] as const async function importFreshEffortModule() { - return import(`./effort.ts?ts=${Date.now()}-${Math.random()}`) + return import(`./effort.js?ts=${Date.now()}-${Math.random()}`) +} + +function restoreProcessEnv(): void { + for (const key of Object.keys(process.env)) { + if (!Object.hasOwn(originalEnv, key)) delete process.env[key] + } + for (const [key, value] of Object.entries(originalEnv)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } } beforeEach(async () => { await acquireSharedMutationLock('utils/effort.test.ts') + mock.module( + './model/modelSupportOverrides.js', + () => realModelSupportOverrides, + ) + for (const key of routingEnvKeys) { + delete process.env[key] + } }) afterEach(() => { @@ -21,7 +79,11 @@ afterEach(() => { mock.restore() mock.module('./auth.js', () => realAuth) mock.module('./thinking.js', () => realThinking) - process.env = { ...originalEnv } + mock.module( + './model/modelSupportOverrides.js', + () => realModelSupportOverrides, + ) + restoreProcessEnv() } finally { releaseSharedMutationLock() } @@ -53,3 +115,326 @@ describe('getDefaultEffortForModel — default-Opus effort gate (#1769)', () => expect(getDefaultEffortForModel('claude-opus-4-1')).toBeUndefined() }) }) + +describe('CLAUDE_CODE_ALWAYS_ENABLE_EFFORT precedence', () => { + test('keeps API-rejecting Claude models excluded across every public effort predicate', async () => { + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + + const { + modelSupportsEffort, + modelSupportsShimReasoningEffort, + modelSupportsWireEffort, + resolveAppliedEffort, + resolveModelReasoningControl, + } = await importFreshEffortModule() + const context = { + apiProvider: 'firstParty' as const, + routeId: 'anthropic', + useRuntimeFallback: false, + } + const rejectedModels = [ + 'claude-3-7-sonnet-20250219', + 'claude-3-5-haiku-20241022', + 'claude-sonnet-4-20250514', + 'claude-sonnet-4-5-20250929', + 'claude-opus-4-20250514', + 'claude-opus-4-1-20250805', + 'claude-haiku-4-5-20251001', + ] + + for (const model of rejectedModels) { + expect(resolveModelReasoningControl(model, context)).toMatchObject({ + supportsReasoning: false, + controllable: false, + source: 'none', + }) + expect(modelSupportsEffort(model, context)).toBe(false) + expect( + modelSupportsShimReasoningEffort( + model, + undefined, + undefined, + context, + ), + ).toBe(false) + expect(modelSupportsWireEffort(model, context)).toBe(false) + expect(resolveAppliedEffort(model, 'medium', context)).toBeUndefined() + } + }) + + test('preserves supported Claude controls and selected wire values', async () => { + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + + const { + modelSupportsEffort, + modelSupportsShimReasoningEffort, + modelSupportsWireEffort, + resolveAppliedEffort, + } = await importFreshEffortModule() + const context = { + apiProvider: 'firstParty' as const, + routeId: 'anthropic', + useRuntimeFallback: false, + } + const supportedModels = [ + ['claude-opus-4-5-20251101', 'low', 'low'], + ['claude-opus-4-6', 'max', 'max'], + ['claude-opus-4-8', 'xhigh', 'xhigh'], + ['claude-sonnet-4-6', 'max', 'high'], + ] as const + + for (const [model, selected, expected] of supportedModels) { + expect(modelSupportsEffort(model, context)).toBe(true) + expect( + modelSupportsShimReasoningEffort( + model, + undefined, + undefined, + context, + ), + ).toBe(true) + expect(modelSupportsWireEffort(model, context)).toBe(true) + expect(resolveAppliedEffort(model, selected, context)).toBe(expected) + } + }) + + test('only force-enables unresolved custom models beyond their provider fallback', async () => { + const { + modelSupportsEffort, + modelSupportsShimReasoningEffort, + modelSupportsWireEffort, + resolveAppliedEffort, + } = await importFreshEffortModule() + const model = 'gateway-custom-model' + const context = { + apiProvider: 'openai' as const, + routeId: 'custom', + useRuntimeFallback: false, + } + const support = () => [ + modelSupportsEffort(model, context), + modelSupportsShimReasoningEffort( + model, + undefined, + undefined, + context, + ), + modelSupportsWireEffort(model, context), + ] + + delete process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT + expect(support()).toEqual([false, false, false]) + expect(resolveAppliedEffort(model, 'medium', context)).toBeUndefined() + + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + expect(support()).toEqual([true, true, true]) + expect(resolveAppliedEffort(model, 'medium', context)).toBe('medium') + }) + + test('uses the scoped routing environment for force enable', async () => { + const { modelSupportsEffort, resolveAppliedEffort } = + await importFreshEffortModule() + const model = 'gateway-custom-model' + const context = { + apiProvider: 'openai' as const, + routeId: 'custom', + useRuntimeFallback: false, + processEnv: { + CLAUDE_CODE_ALWAYS_ENABLE_EFFORT: '1', + } as NodeJS.ProcessEnv, + } + + delete process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT + expect(modelSupportsEffort(model, context)).toBe(true) + expect(resolveAppliedEffort(model, 'medium', context)).toBe('medium') + + process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT = '1' + context.processEnv = {} + expect(modelSupportsEffort(model, context)).toBe(false) + expect(resolveAppliedEffort(model, 'medium', context)).toBeUndefined() + }) + + test('uses the scoped environment for compatibility and catalog route fallbacks', async () => { + const { resolveModelReasoningControl } = await importFreshEffortModule() + const scopedOpenAIEnv = { + CLAUDE_CODE_USE_OPENAI: '1', + OPENAI_API_KEY: 'scoped-key', + OPENAI_BASE_URL: 'https://api.openai.com/v1', + } as NodeJS.ProcessEnv + const context = { + apiProvider: 'openai' as const, + processEnv: scopedOpenAIEnv, + } + const noControl = { + supportsReasoning: false, + controllable: false, + source: 'none', + } + + process.env.LONGCAT_API_KEY = 'ambient-key' + expect(resolveModelReasoningControl('gateway-custom-model', context)).toMatchObject( + noControl, + ) + + delete process.env.LONGCAT_API_KEY + process.env.ZAI_API_KEY = 'ambient-key' + expect( + resolveModelReasoningControl('glm-5.2', { + ...context, + openaiShimConfig: {}, + }), + ).toMatchObject(noControl) + + delete process.env.ZAI_API_KEY + process.env.XAI_API_KEY = 'ambient-key' + expect(resolveModelReasoningControl('grok-4.6', context)).toMatchObject( + noControl, + ) + + delete process.env.XAI_API_KEY + process.env.OPENAI_BASE_URL = 'https://api.openai.com/v1' + expect( + resolveModelReasoningControl('gpt-5.6', { + apiProvider: 'openai', + processEnv: { + CLAUDE_CODE_USE_OPENAI: '1', + OPENAI_API_KEY: 'scoped-key', + OPENAI_API_BASE: 'https://gateway.example.test/v1', + }, + supportsCodexReasoningEffort: false, + }), + ).toMatchObject(noControl) + }) + + test('ambient custom-route predicates do not advertise native transport effort', async () => { + delete process.env.CLAUDE_CODE_USE_GEMINI + delete process.env.GEMINI_API_KEY + process.env.CLAUDE_CODE_USE_OPENAI = '1' + process.env.OPENAI_API_KEY = 'test-key' + process.env.OPENAI_BASE_URL = 'https://gateway.example.test/v1' + process.env.OPENAI_MODEL = 'claude-opus-4-5' + + const { modelSupportsEffort, resolveAppliedEffort } = + await importFreshEffortModule() + + for (const model of ['claude-opus-4-5', 'gemini-3-pro']) { + expect(modelSupportsEffort(model)).toBe(false) + expect(resolveAppliedEffort(model, 'medium')).toBeUndefined() + } + expect(modelSupportsEffort('gpt-5.4')).toBe(true) + expect(resolveAppliedEffort('gpt-5.4', 'medium')).toBe('medium') + }) +}) + +describe('configured third-party effort precedence', () => { + test('supersedes descriptive LongCat catalog capabilities without bypassing route contracts', async () => { + ensureIntegrationsLoaded() + const longCatEntry = getCatalogEntriesForRoute('longcat').find( + entry => entry.apiName === 'LongCat-2.0', + )! + const { + getAvailableEffortLevels, + modelSupportsEffort, + modelSupportsWireEffort, + resolveAppliedEffort, + resolveModelReasoningControl, + } = await importFreshEffortModule() + const context = { + apiProvider: 'openai' as const, + routeId: 'longcat', + useRuntimeFallback: false, + } + + expect(resolveModelReasoningControl('LongCat-2.0', context)).toMatchObject({ + supportsReasoning: true, + controllable: false, + source: 'capability', + levels: [], + }) + + process.env.ANTHROPIC_DEFAULT_SONNET_MODEL = 'LongCat-2.0' + process.env.ANTHROPIC_DEFAULT_SONNET_MODEL_SUPPORTED_CAPABILITIES = + 'effort,max_effort,xhigh_effort' + + expect(resolveModelReasoningControl('LongCat-2.0', context)).toMatchObject({ + supportsReasoning: true, + controllable: true, + source: 'capability', + levels: ['low', 'medium', 'high', 'xhigh', 'max'], + wireFormat: 'reasoning_effort', + }) + expect(modelSupportsEffort('LongCat-2.0', context)).toBe(true) + expect(modelSupportsWireEffort('LongCat-2.0', context)).toBe(true) + expect(getAvailableEffortLevels('LongCat-2.0', context)).toEqual([ + 'low', + 'medium', + 'high', + 'xhigh', + 'max', + ]) + expect(resolveAppliedEffort('LongCat-2.0', 'max', context)).toBe('max') + + const transportVetoContext = { + ...context, + openaiShimConfig: { removeBodyFields: ['reasoning_effort'] }, + } + expect( + resolveModelReasoningControl('LongCat-2.0', transportVetoContext), + ).toMatchObject({ + controllable: false, + source: 'compat', + }) + expect( + resolveAppliedEffort('LongCat-2.0', 'max', transportVetoContext), + ).toBeUndefined() + + const explicitMetadataContext = { + ...context, + catalogEntries: [ + { + ...longCatEntry, + reasoning: { + mode: 'always-on' as const, + wireFormat: 'none' as const, + }, + }, + ], + } + expect( + resolveModelReasoningControl('LongCat-2.0', explicitMetadataContext), + ).toMatchObject({ + supportsReasoning: true, + controllable: false, + source: 'metadata', + wireFormat: 'none', + }) + }) + + test('preserves the legacy default for a configured override', async () => { + process.env.ANTHROPIC_DEFAULT_OPUS_MODEL = 'claude-opus-4-6' + process.env.ANTHROPIC_DEFAULT_OPUS_MODEL_SUPPORTED_CAPABILITIES = 'effort' + mock.module('./auth.js', () => ({ + ...realAuth, + isProSubscriber: () => true, + isMaxSubscriber: () => false, + isTeamSubscriber: () => false, + })) + mock.module('./thinking.js', () => ({ + ...realThinking, + isUltrathinkEnabled: () => false, + })) + + const { getDefaultEffortForModel, resolveAppliedEffort } = + await importFreshEffortModule() + const context = { + apiProvider: 'openai' as const, + routeId: 'custom', + useRuntimeFallback: false, + } + + expect(getDefaultEffortForModel('claude-opus-4-6', context)).toBe('medium') + expect( + resolveAppliedEffort('claude-opus-4-6', undefined, context), + ).toBe('medium') + }) +}) diff --git a/src/utils/effort.ts b/src/utils/effort.ts index dbad42dab..3c65ee457 100644 --- a/src/utils/effort.ts +++ b/src/utils/effort.ts @@ -75,6 +75,11 @@ type OpenAIShimReasoningSupportContext = { useRuntimeFallback?: boolean } +type ReasoningCompatibilityOverrides = { + thinkingRequestFormat?: OpenAIShimThinkingRequestFormat + removeBodyFields?: string[] +} + export type ReasoningControlContext = OpenAIShimReasoningSupportContext & { apiProvider?: ReturnType supportsCodexReasoningEffort?: boolean | ((model: string) => boolean) @@ -219,6 +224,7 @@ function resolveCompatibilityWireFormat( thinkingRequestFormat?: OpenAIShimThinkingRequestFormat, routeIdOverride?: string | null, useRuntimeFallback = true, + processEnv: NodeJS.ProcessEnv = process.env, ): ReasoningWireFormat | undefined { if (thinkingRequestFormat === 'deepseek-compatible') { return 'deepseek_compatible' @@ -233,7 +239,7 @@ function resolveCompatibilityWireFormat( const routeId = routeIdOverride !== undefined ? routeIdOverride : useRuntimeFallback - ? resolveActiveRouteIdFromEnv(process.env) + ? resolveActiveRouteIdFromEnv(processEnv) : undefined if (!routeId || routeId === 'anthropic' || routeId === 'openai') { return undefined @@ -254,9 +260,10 @@ function resolveCompatibilityReasoningControl( context?: ReasoningControlContext, ): ReasoningControlResolution | undefined { const useRuntimeFallback = context?.useRuntimeFallback ?? true + const processEnv = context?.processEnv ?? process.env const runtimeShimConfig = context?.openaiShimConfig ?? (useRuntimeFallback && thinkingRequestFormat === undefined && removeBodyFields === undefined ? resolveOpenAIShimRuntimeContext({ - processEnv: process.env, + processEnv, model, }).openaiShimConfig : undefined) @@ -264,20 +271,29 @@ function resolveCompatibilityReasoningControl( thinkingRequestFormat ?? runtimeShimConfig?.thinkingRequestFormat const resolvedRemoveBodyFields = removeBodyFields ?? runtimeShimConfig?.removeBodyFields + if ( + resolvedThinkingRequestFormat === 'none' || + resolvedRemoveBodyFields?.includes('reasoning_effort') + ) { + return { + supportsReasoning: false, + controllable: false, + levels: [], + source: 'compat', + } + } const wireFormat = resolveCompatibilityWireFormat( model, resolvedThinkingRequestFormat, context?.routeId, useRuntimeFallback, + processEnv, ) if (!wireFormat) { return undefined } if (wireFormat === 'deepseek_compatible') { - if (resolvedRemoveBodyFields?.includes('reasoning_effort')) { - return undefined - } return { supportsReasoning: true, controllable: true, @@ -290,9 +306,7 @@ function resolveCompatibilityReasoningControl( } if (wireFormat === 'zai_compatible') { - const reasoningEffortStripped = - resolvedRemoveBodyFields?.includes('reasoning_effort') === true - const levels: EffortLevel[] = supportsZaiReasoningEffort(model) && !reasoningEffortStripped + const levels: EffortLevel[] = supportsZaiReasoningEffort(model) ? ['high', 'xhigh'] : ['high'] return { @@ -316,11 +330,12 @@ function resolveCatalogReasoningMetadata( capabilities?: CapabilityFlags reasoning?: ReasoningControlMetadata } | undefined { + const processEnv = context?.processEnv ?? process.env const routeId = context?.routeId !== undefined ? context.routeId : context?.useRuntimeFallback === false ? undefined - : resolveActiveRouteIdFromEnv(process.env) + : resolveActiveRouteIdFromEnv(processEnv) if (!routeId || routeId === 'anthropic') { return undefined } @@ -337,7 +352,7 @@ function resolveCatalogReasoningMetadata( const entries = context?.catalogEntries ?? getCatalogEntriesForRoute(routeId) let entry = entries.find(matchesModel) const fallbackBaseUrl = - context?.baseUrl ?? context?.processEnv?.OPENAI_BASE_URL ?? process.env.OPENAI_BASE_URL ?? process.env.OPENAI_API_BASE + context?.baseUrl ?? processEnv.OPENAI_BASE_URL ?? processEnv.OPENAI_API_BASE if ( !entry && routeId === 'custom' && @@ -416,21 +431,103 @@ function resolveMetadataReasoningControl( } } +function resolveConfigured3PReasoningControl( + model: string, + context?: ReasoningControlContext, +): ReasoningControlResolution | undefined { + const apiProvider = getReasoningApiProvider(context) + if (get3PModelCapabilityOverride(model, 'effort', apiProvider) !== true) { + return undefined + } + + const levels: EffortLevel[] = ['low', 'medium', 'high'] + if ( + get3PModelCapabilityOverride(model, 'xhigh_effort', apiProvider) === true + ) { + levels.push('xhigh') + } + if ( + get3PModelCapabilityOverride(model, 'max_effort', apiProvider) === true + ) { + levels.push('max') + } + + return { + supportsReasoning: true, + controllable: true, + mode: 'levels', + levels, + defaultLevel: getLegacyDefaultEffortForModel(model, context), + wireFormat: 'reasoning_effort', + source: 'capability', + } +} + +type NativeLegacyEffortTransport = 'anthropic' | 'gemini' + +function resolveNativeLegacyEffortTransport( + model: string, + context?: ReasoningControlContext, +): NativeLegacyEffortTransport | undefined { + const useRuntimeFallback = context?.useRuntimeFallback ?? true + const runtimeShimConfig = context?.openaiShimConfig ?? ( + useRuntimeFallback + ? resolveOpenAIShimRuntimeContext({ + processEnv: context?.processEnv ?? process.env, + baseUrl: context?.baseUrl, + model, + }).openaiShimConfig + : undefined + ) + const endpointPath = runtimeShimConfig?.endpointPath + if (endpointPath === '/messages') return 'anthropic' + if (endpointPath?.startsWith('/models/gemini-')) return 'gemini' + + const apiProvider = getReasoningApiProvider(context) + if ( + apiProvider === 'firstParty' || + apiProvider === 'bedrock' || + apiProvider === 'vertex' || + apiProvider === 'foundry' || + apiProvider === 'github' + ) { + return 'anthropic' + } + return apiProvider === 'gemini' ? 'gemini' : undefined +} + +function modelMatchesNativeLegacyTransport( + model: string, + transport: NativeLegacyEffortTransport | undefined, +): boolean { + const normalized = model.toLowerCase() + return transport === 'anthropic' + ? normalized.includes('haiku') || + normalized.includes('sonnet') || + normalized.includes('opus') + : transport === 'gemini' && normalized.includes('gemini-') +} + function legacyModelSupportsEffort( model: string, context?: ReasoningControlContext, ): boolean { const m = model.toLowerCase() - if (isEnvTruthy(process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT)) { - return true - } - const supported3P = get3PModelCapabilityOverride(model, 'effort') + const supported3P = get3PModelCapabilityOverride( + model, + 'effort', + getReasoningApiProvider(context), + ) if (supported3P !== undefined) { return supported3P } - if (modelUsesOpenAIEffort(model, context) && modelSupportsCodexReasoningEffort(model, context)) { + if ( + modelUsesOpenAIEffort(model, context) && + modelSupportsCodexReasoningEffort(model, context) + ) { return true } + const nativeTransport = resolveNativeLegacyEffortTransport(model, context) // Claude 4 models that support effort. Mirrors the Anthropic /messages // shim's isAdaptive || isOpus45 set (openaiShim.ts:2292-2297) — only // these models serialize low/medium as anthropicBody.effort. Older @@ -438,19 +535,35 @@ function legacyModelSupportsEffort( // high/max, so advertising effort for them would silently drop // low/medium on the wire. The substring match also covers prefix // variations (e.g. `claude-opus-4-7`, `opencode-claude-opus-4-8`). - if (m.includes('opus-4-5') || m.includes('opus-4-6') || + if ( + nativeTransport === 'anthropic' && + (m.includes('opus-4-5') || m.includes('opus-4-6') || m.includes('opus-4-7') || m.includes('opus-4-8') || - m.includes('sonnet-4-6')) { + m.includes('sonnet-4-6')) + ) { return true } // OpenCode Gemini models that support thinking via /models/gemini-* endpoint - if (m.includes('gemini-3')) { + if (nativeTransport === 'gemini' && m.includes('gemini-3')) { return true } - // Exclude any other known legacy models (haiku, older opus/sonnet variants) - if (m.includes('haiku') || m.includes('sonnet') || m.includes('opus')) { + // Native model names need an authorized native transport before force-enable + // or provider defaults may add an incompatible field to a generic shim. + if ( + m.includes('haiku') || + m.includes('sonnet') || + m.includes('opus') || + m.includes('gemini-') + ) { return false } + if ( + isEnvTruthy( + (context?.processEnv ?? process.env).CLAUDE_CODE_ALWAYS_ENABLE_EFFORT, + ) + ) { + return true + } // IMPORTANT: Do not change the default effort support without notifying // the model launch DRI and research. This is a sensitive setting that can @@ -489,17 +602,42 @@ function resolveLegacyReasoningControl( export function resolveModelReasoningControl( model: string, context?: ReasoningControlContext, + compatibilityOverrides?: ReasoningCompatibilityOverrides, ): ReasoningControlResolution { const metadata = resolveMetadataReasoningControl(model, context) - if (metadata?.source === 'metadata') { + const compatibility = resolveCompatibilityReasoningControl( + model, + compatibilityOverrides?.thinkingRequestFormat, + compatibilityOverrides?.removeBodyFields, + context, + ) + if (compatibility && !compatibility.controllable) { + return compatibility + } + if (metadata?.source === 'metadata' || metadata?.supportsReasoning === false) { return metadata } - const compatibility = resolveCompatibilityReasoningControl(model, undefined, undefined, context) if (compatibility) { return compatibility } + const configured3P = resolveConfigured3PReasoningControl(model, context) + if (configured3P) { + return configured3P + } + + const nativeTransport = resolveNativeLegacyEffortTransport(model, context) + if ( + metadata?.source === 'capability' && + modelMatchesNativeLegacyTransport(model, nativeTransport) + ) { + const nativeLegacy = resolveLegacyReasoningControl(model, context) + if (nativeLegacy.controllable) { + return nativeLegacy + } + } + if (metadata) { return metadata } @@ -509,13 +647,6 @@ export function resolveModelReasoningControl( // @[MODEL LAUNCH]: Add the new model to the allowlist if it supports the effort parameter. export function modelSupportsEffort(model: string, context?: ReasoningControlContext): boolean { - if (isEnvTruthy(process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT)) { - return true - } - const supported3P = get3PModelCapabilityOverride(model, 'effort') - if (supported3P !== undefined) { - return supported3P - } return resolveModelReasoningControl(model, context).controllable } @@ -525,63 +656,15 @@ export function modelSupportsShimReasoningEffort( removeBodyFields?: string[], context?: ReasoningControlContext, ): boolean { - if (isEnvTruthy(process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT)) { - return true - } - const supported3P = get3PModelCapabilityOverride(model, 'effort') - if (supported3P !== undefined) { - return supported3P - } - - const metadata = resolveMetadataReasoningControl( + const control = resolveModelReasoningControl( model, context, + { thinkingRequestFormat, removeBodyFields }, ) - if (metadata?.source === 'metadata') { - return Boolean(metadata.controllable && metadataWireFormatSupportsEffort(metadata.wireFormat)) - } - - const compatibility = resolveCompatibilityReasoningControl( - model, - thinkingRequestFormat, - removeBodyFields, - context, - ) - if (compatibility) { - return compatibility.controllable - } - - if ( - context?.routeId && - (context.routeId === 'openai' || context.routeId === 'codex') && - !removeBodyFields?.includes('reasoning_effort') - ) { - return modelSupportsCodexReasoningEffort(model, context) - } - - if (context?.useRuntimeFallback === false) { - if ( - context.routeId == null && - thinkingRequestFormat === undefined && - !removeBodyFields?.includes('reasoning_effort') - ) { - return resolveLegacyReasoningControl(model, context).controllable - } - return false - } - - const control = metadata ?? resolveLegacyReasoningControl(model, context) return Boolean(control.controllable && metadataWireFormatSupportsEffort(control.wireFormat)) } export function modelSupportsWireEffort(model: string, context?: ReasoningControlContext): boolean { - if (isEnvTruthy(process.env.CLAUDE_CODE_ALWAYS_ENABLE_EFFORT)) { - return true - } - const supported3P = get3PModelCapabilityOverride(model, 'effort') - if (supported3P !== undefined) { - return supported3P - } return modelSupportsShimReasoningEffort(model, undefined, undefined, context) } @@ -701,8 +784,15 @@ export function resolveOpenAIShimReasoningRequestPlan(options: { // @[MODEL LAUNCH]: Add the new model to the allowlist if it supports 'max' effort. // Per API docs, 'max' is supported on the recent Opus models (4.8/4.7/4.6) for // public models — other models return an error. -function legacyModelSupportsMaxEffort(model: string): boolean { - const supported3P = get3PModelCapabilityOverride(model, 'max_effort') +function legacyModelSupportsMaxEffort( + model: string, + context?: ReasoningControlContext, +): boolean { + const supported3P = get3PModelCapabilityOverride( + model, + 'max_effort', + getReasoningApiProvider(context), + ) if (supported3P !== undefined) { return supported3P } @@ -725,7 +815,11 @@ function legacyModelSupportsXHighEffort( if (!legacyModelSupportsEffort(model, context)) { return false } - const supported3P = get3PModelCapabilityOverride(model, 'xhigh_effort') + const supported3P = get3PModelCapabilityOverride( + model, + 'xhigh_effort', + getReasoningApiProvider(context), + ) if (supported3P !== undefined) { return supported3P } @@ -787,7 +881,7 @@ function getLegacyAvailableEffortLevels( if (legacyModelSupportsXHighEffort(model, context)) { levels.push('xhigh') } - if (legacyModelSupportsMaxEffort(model)) { + if (legacyModelSupportsMaxEffort(model, context)) { levels.push('max') } if ( @@ -818,7 +912,7 @@ export function modelSupportsMaxEffort(model: string, context?: ReasoningControl if (control.source === 'metadata' || control.source === 'capability' || control.source === 'compat') { return control.levels.includes('max') } - return legacyModelSupportsMaxEffort(model) + return legacyModelSupportsMaxEffort(model, context) } export function modelSupportsXHighEffort(model: string, context?: ReasoningControlContext): boolean { diff --git a/src/utils/model/modelSupportOverrides.ts b/src/utils/model/modelSupportOverrides.ts index 164249327..eb91259db 100644 --- a/src/utils/model/modelSupportOverrides.ts +++ b/src/utils/model/modelSupportOverrides.ts @@ -27,7 +27,9 @@ const TIERS = [ function buildCapabilityOverrideCacheKey( model: string, capability: ModelCapabilityOverride, + apiProvider?: ReturnType, ): string { + const resolvedApiProvider = apiProvider ?? getAPIProvider() const envParts = TIERS.flatMap(tier => [ process.env[tier.modelEnvVar] ?? '', process.env[tier.capabilitiesEnvVar] ?? '', @@ -36,7 +38,7 @@ function buildCapabilityOverrideCacheKey( return [ model.toLowerCase(), capability, - getAPIProvider(), + resolvedApiProvider, process.env.ANTHROPIC_BASE_URL ?? '', process.env.USER_TYPE ?? '', ...envParts, @@ -48,9 +50,14 @@ function buildCapabilityOverrideCacheKey( * the pinned ANTHROPIC_DEFAULT_*_MODEL env vars. */ export const get3PModelCapabilityOverride = memoize( - (model: string, capability: ModelCapabilityOverride): boolean | undefined => { + ( + model: string, + capability: ModelCapabilityOverride, + apiProvider?: ReturnType, + ): boolean | undefined => { + const resolvedApiProvider = apiProvider ?? getAPIProvider() if ( - getAPIProvider() === 'firstParty' && + resolvedApiProvider === 'firstParty' && isFirstPartyAnthropicBaseUrl() ) { return undefined