From 19bb39c8845a6920bb04ad8d70652ad520861b87 Mon Sep 17 00:00:00 2001 From: JATMN Date: Wed, 21 Oct 2026 21:00:41 -0700 Subject: [PATCH] test: consolidate gateway catalog tests into per-gateway contract tables (#2325) * test: consolidate gateway catalog tests into per-gateway contract tables * test: assert opencode-go catalog excludes removed model ids --- src/integrations/gateways/api-route.test.ts | 63 +++-- src/integrations/gateways/apismart.test.ts | 41 +-- src/integrations/gateways/commandcode.test.ts | 92 +++---- src/integrations/gateways/concentrate.test.ts | 62 +++-- src/integrations/gateways/custom.test.ts | 29 +++ .../gateways/gitlawb-opengateway.test.ts | 27 +- src/integrations/gateways/gmi-cloud.test.ts | 62 +++-- src/integrations/gateways/llmtr.test.ts | 80 +++--- .../gateways/merge-gateway.test.ts | 52 ++-- src/integrations/gateways/nvidia-nim.test.ts | 25 +- src/integrations/gateways/opencode.test.ts | 241 +++++------------- src/integrations/gateways/openrouter.test.ts | 27 +- .../gateways/xiaomi-mimo-token.test.ts | 40 ++- src/test/gatewayCatalogTestHelpers.ts | 132 ++++++++++ 14 files changed, 557 insertions(+), 416 deletions(-) create mode 100644 src/test/gatewayCatalogTestHelpers.ts diff --git a/src/integrations/gateways/api-route.test.ts b/src/integrations/gateways/api-route.test.ts index 98b4824f5..3014ad26e 100644 --- a/src/integrations/gateways/api-route.test.ts +++ b/src/integrations/gateways/api-route.test.ts @@ -1,4 +1,4 @@ -import { expect, test } from 'bun:test' +import { describe, expect, test } from 'bun:test' import { routeForPreset } from '../compatibility.js' import { getProviderPresetUiMetadata } from '../providerUiMetadata.js' @@ -7,32 +7,47 @@ import { resolveRouteCredentialValue, resolveRouteIdFromBaseUrl, } from '../routeMetadata.js' -import gateway, { mapApiRouteModel } from './api-route.js' +import { mapApiRouteModel } from './api-route.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' -test('API Route uses a dedicated hybrid OpenAI-compatible gateway contract', () => { - expect(gateway.defaultBaseUrl).toBe('https://global.api-route.com/v1') - expect(gateway.defaultModel).toBe('claude-sonnet-4-6') - expect(gateway.setup.credentialEnvVars).toEqual(['API_ROUTE_API_KEY']) - expect(gateway.setup.dedicatedCredentialsOnly).toBe(false) - expect(gateway.preset?.apiKeyEnvVars).toEqual(['API_ROUTE_API_KEY']) - expect(gateway.validation).toMatchObject({ - kind: 'credential-env', - routing: { matchDefaultBaseUrl: true }, - credentialEnvVars: ['API_ROUTE_API_KEY', 'OPENAI_API_KEYS', 'OPENAI_API_KEY'], - }) - expect(gateway.transportConfig).toEqual({ - kind: 'openai-compatible', - openaiShim: { - supportsApiFormatSelection: false, - supportsAuthHeaders: false, - maxTokensField: 'max_tokens', +describe('api-route gateway', () => { + describe.each([ + { + routeId: 'api-route', + expected: { + baseUrl: 'https://global.api-route.com/v1', + defaultModel: 'claude-sonnet-4-6', + setup: { + credentialEnvVars: ['API_ROUTE_API_KEY'], + dedicatedCredentialsOnly: false, + }, + preset: { apiKeyEnvVars: ['API_ROUTE_API_KEY'] }, + validation: { + kind: 'credential-env', + routing: { matchDefaultBaseUrl: true }, + credentialEnvVars: [ + 'API_ROUTE_API_KEY', + 'OPENAI_API_KEYS', + 'OPENAI_API_KEY', + ], + }, + openaiShim: { + supportsApiFormatSelection: false, + supportsAuthHeaders: false, + maxTokensField: 'max_tokens', + }, + catalogSource: 'hybrid', + discovery: { requiresAuth: true, mapModel: mapApiRouteModel }, + } satisfies GatewayCatalogExpectations, }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + }) }) - expect(gateway.catalog?.source).toBe('hybrid') - expect(gateway.catalog?.discovery?.requiresAuth).toBe(true) - expect(gateway.catalog?.models?.map(model => model.apiName)).toEqual([ - 'claude-sonnet-4-6', - ]) }) test('API Route preset uses the existing generic profile path', () => { diff --git a/src/integrations/gateways/apismart.test.ts b/src/integrations/gateways/apismart.test.ts index 2086ddd17..342d95ff3 100644 --- a/src/integrations/gateways/apismart.test.ts +++ b/src/integrations/gateways/apismart.test.ts @@ -1,29 +1,32 @@ import { describe, expect, test } from 'bun:test' import apismart from './apismart.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' const mapModel = apismart.catalog?.discovery?.mapModel describe('apismart gateway', () => { - test('uses hybrid discovery with dedicated credentials', () => { - expect(apismart.id).toBe('apismart') - expect(apismart.catalog?.source).toBe('hybrid') - expect(apismart.catalog?.discovery?.kind).toBe('openai-compatible') - expect(apismart.catalog?.discovery?.requiresAuth).toBe(true) - expect(apismart.setup.dedicatedCredentialsOnly).toBe(true) - expect(apismart.setup.credentialEnvVars).toEqual(['APISMART_API_KEY']) - expect(apismart.defaultBaseUrl).toBe('https://gw.apismart.ai/v1') - expect(apismart.defaultModel).toBe('DEEPSEEK_V4_FLASH') - expect(mapModel).toBeDefined() - }) - - test('curated catalog keeps ApiSmart case-sensitive model ids', () => { - const models = apismart.catalog?.models ?? [] - expect(models.some(model => model.apiName === 'DEEPSEEK_V4_FLASH')).toBe( - true, - ) - expect(models.some(model => model.apiName === 'GLM_5.2')).toBe(true) - expect(models.some(model => model.apiName === 'QWEN_3_7_MAX')).toBe(true) + describe.each([ + { + routeId: 'apismart', + expected: { + baseUrl: 'https://gw.apismart.ai/v1', + defaultModel: 'DEEPSEEK_V4_FLASH', + setup: { + credentialEnvVars: ['APISMART_API_KEY'], + dedicatedCredentialsOnly: true, + }, + catalogSource: 'hybrid', + discovery: { requiresAuth: true }, + } satisfies GatewayCatalogExpectations, + }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + }) }) test('mapModel keeps chat models and drops image/video ids', () => { diff --git a/src/integrations/gateways/commandcode.test.ts b/src/integrations/gateways/commandcode.test.ts index 8f7c2e0cc..a76cbe288 100644 --- a/src/integrations/gateways/commandcode.test.ts +++ b/src/integrations/gateways/commandcode.test.ts @@ -1,4 +1,4 @@ -import { expect, test } from 'bun:test' +import { describe, expect, test } from 'bun:test' import { routeForPreset } from '../compatibility.js' import { getProviderPresetUiMetadata } from '../providerUiMetadata.js' @@ -7,54 +7,60 @@ import { resolveRouteCredentialValue, resolveRouteIdFromBaseUrl, } from '../routeMetadata.js' -import gateway, { +import { getCommandcodeChatCompletionsModelError, mapCommandcodeModel, } from './commandcode.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' -test('Command Code uses a dedicated hybrid OpenAI-compatible gateway contract', () => { - expect(gateway.defaultBaseUrl).toBe( - 'https://api.commandcode.ai/provider/v1', - ) - expect(gateway.defaultModel).toBe('deepseek/deepseek-v4-flash') - expect(gateway.setup.credentialEnvVars).toEqual([ - 'CMD_API_KEY', - 'COMMANDCODE_API_KEY', - 'COMMAND_CODE_API_KEY', - ]) - expect(gateway.setup.dedicatedCredentialsOnly).toBe(true) - expect(gateway.preset?.apiKeyEnvVars).toEqual([ - 'CMD_API_KEY', - 'COMMANDCODE_API_KEY', - 'COMMAND_CODE_API_KEY', - ]) - expect(gateway.validation).toMatchObject({ - kind: 'credential-env', - routing: { matchDefaultBaseUrl: true }, - credentialEnvVars: [ - 'CMD_API_KEY', - 'COMMANDCODE_API_KEY', - 'COMMAND_CODE_API_KEY', - ], - }) - expect(gateway.transportConfig).toEqual({ - kind: 'openai-compatible', - openaiShim: { - requiredApiFormat: 'chat_completions', - supportsAuthHeaders: false, - maxTokensField: 'max_tokens', +describe('commandcode gateway', () => { + describe.each([ + { + routeId: 'commandcode', + expected: { + baseUrl: 'https://api.commandcode.ai/provider/v1', + defaultModel: 'deepseek/deepseek-v4-flash', + setup: { + credentialEnvVars: [ + 'CMD_API_KEY', + 'COMMANDCODE_API_KEY', + 'COMMAND_CODE_API_KEY', + ], + dedicatedCredentialsOnly: true, + }, + preset: { + apiKeyEnvVars: [ + 'CMD_API_KEY', + 'COMMANDCODE_API_KEY', + 'COMMAND_CODE_API_KEY', + ], + }, + validation: { + kind: 'credential-env', + routing: { matchDefaultBaseUrl: true }, + credentialEnvVars: [ + 'CMD_API_KEY', + 'COMMANDCODE_API_KEY', + 'COMMAND_CODE_API_KEY', + ], + }, + openaiShim: { + requiredApiFormat: 'chat_completions', + supportsAuthHeaders: false, + maxTokensField: 'max_tokens', + }, + catalogSource: 'hybrid', + discovery: { requiresAuth: false, mapModel: mapCommandcodeModel }, + } satisfies GatewayCatalogExpectations, }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + }) }) - expect(gateway.catalog?.source).toBe('hybrid') - expect(gateway.catalog?.discovery?.requiresAuth).toBe(false) - expect(gateway.catalog?.models?.map(model => model.apiName)).toEqual([ - 'deepseek/deepseek-v4-flash', - 'deepseek/deepseek-v4-pro', - 'gpt-5.6-sol', - 'moonshotai/Kimi-K2.5', - 'MiniMaxAI/MiniMax-M3', - 'z-ai/glm-5.3-flash', - ]) }) test('Command Code preset uses the existing generic profile path', () => { diff --git a/src/integrations/gateways/concentrate.test.ts b/src/integrations/gateways/concentrate.test.ts index 963c90ac9..df325d497 100644 --- a/src/integrations/gateways/concentrate.test.ts +++ b/src/integrations/gateways/concentrate.test.ts @@ -1,37 +1,45 @@ import { describe, expect, test } from 'bun:test' import concentrate from './concentrate.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' const mapModel = concentrate.catalog?.discovery?.mapModel describe('concentrate gateway', () => { - test('uses dynamic discovery and supports fallback auth', () => { - expect(concentrate.id).toBe('concentrate') - expect(concentrate.label).toBe('Concentrate') - expect(concentrate.category).toBe('aggregating') - expect(concentrate.defaultBaseUrl).toBe('https://api.concentrate.ai/v1') - expect(concentrate.defaultModel).toBe('deepseek-v4-flash') - expect(concentrate.supportsModelRouting).toBe(true) - - expect(concentrate.setup.requiresAuth).toBe(true) - expect(concentrate.setup.authMode).toBe('api-key') - expect(concentrate.setup.credentialEnvVars).toEqual(['CONCENTRATE_API_KEY']) - expect(concentrate.setup.dedicatedCredentialsOnly).toBeUndefined() - - expect(concentrate.catalog?.source).toBe('dynamic') - expect(concentrate.catalog?.discovery?.kind).toBe('openai-compatible') - expect(concentrate.catalog?.discovery?.requiresAuth).toBe(false) - - expect(concentrate.validation?.kind).toBe('credential-env') - expect( - (concentrate.validation as { credentialEnvVars: string[] }).credentialEnvVars, - ).toEqual([ - 'CONCENTRATE_API_KEY', - 'OPENAI_API_KEYS', - 'OPENAI_API_KEY', - ]) - - expect(mapModel).toBeDefined() + describe.each([ + { + routeId: 'concentrate', + expected: { + baseUrl: 'https://api.concentrate.ai/v1', + defaultModel: 'deepseek-v4-flash', + category: 'aggregating', + supportsModelRouting: true, + setup: { + requiresAuth: true, + authMode: 'api-key', + credentialEnvVars: ['CONCENTRATE_API_KEY'], + dedicatedCredentialsOnly: undefined, + }, + validation: { + kind: 'credential-env', + credentialEnvVars: [ + 'CONCENTRATE_API_KEY', + 'OPENAI_API_KEYS', + 'OPENAI_API_KEY', + ], + }, + catalogSource: 'dynamic', + discovery: { requiresAuth: false }, + allowEmptyCatalog: true, + } satisfies GatewayCatalogExpectations, + }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + }) }) test('mapModel keeps chat models and drops non-chat ids', () => { diff --git a/src/integrations/gateways/custom.test.ts b/src/integrations/gateways/custom.test.ts index 460e95f2e..16b2111ef 100644 --- a/src/integrations/gateways/custom.test.ts +++ b/src/integrations/gateways/custom.test.ts @@ -1,5 +1,9 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test' import { _clearRegistryForTesting, ensureIntegrationsLoaded, getCatalogForGateway } from '../index.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' describe('custom gateway', () => { beforeEach(() => { @@ -11,6 +15,31 @@ describe('custom gateway', () => { _clearRegistryForTesting() }) + describe.each([ + { + routeId: 'custom', + expected: { + baseUrl: undefined, + defaultModel: 'llama3.1:8b', + setup: { requiresAuth: false, credentialEnvVars: undefined }, + preset: { + fallbackBaseUrl: 'http://localhost:11434/v1', + apiKeyEnvVars: ['OPENAI_API_KEYS', 'OPENAI_API_KEY'], + }, + openaiShim: { + supportsApiFormatSelection: true, + supportsAuthHeaders: true, + }, + catalogSource: 'hybrid', + allowEmptyCatalog: true, + } satisfies GatewayCatalogExpectations, + }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + }) + }) + test('discovers /v1/models and maps context_length to contextWindow', async () => { const catalog = getCatalogForGateway('custom') expect(catalog?.source).toBe('hybrid') diff --git a/src/integrations/gateways/gitlawb-opengateway.test.ts b/src/integrations/gateways/gitlawb-opengateway.test.ts index e178758c0..665a31d6a 100644 --- a/src/integrations/gateways/gitlawb-opengateway.test.ts +++ b/src/integrations/gateways/gitlawb-opengateway.test.ts @@ -1,17 +1,24 @@ import { describe, expect, test } from 'bun:test' import opengateway, { mapOpenGatewayModel } from './gitlawb-opengateway.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' describe('gitlawb-opengateway live model mapping', () => { - test('uses hybrid discovery against the public models list', () => { - expect(opengateway.catalog?.source).toBe('hybrid') - expect(opengateway.catalog?.discovery).toEqual( - expect.objectContaining({ - kind: 'openai-compatible', - requiresAuth: false, - }), - ) - expect(opengateway.catalog?.discovery?.mapModel).toBe(mapOpenGatewayModel) - expect(opengateway.startup?.probeReadiness).toBeUndefined() + describe.each([ + { + routeId: 'gitlawb-opengateway', + expected: { + catalogSource: 'hybrid', + discovery: { requiresAuth: false, mapModel: mapOpenGatewayModel }, + } satisfies GatewayCatalogExpectations, + }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + expect(opengateway.startup?.probeReadiness).toBeUndefined() + }) }) test('maps gateway routes including auto and free models', () => { diff --git a/src/integrations/gateways/gmi-cloud.test.ts b/src/integrations/gateways/gmi-cloud.test.ts index 8e43e1652..eb313c23c 100644 --- a/src/integrations/gateways/gmi-cloud.test.ts +++ b/src/integrations/gateways/gmi-cloud.test.ts @@ -1,4 +1,4 @@ -import { expect, test } from 'bun:test' +import { describe, expect, test } from 'bun:test' import { routeForPreset } from '../compatibility.js' import { getProviderPresetUiMetadata } from '../providerUiMetadata.js' @@ -9,35 +9,43 @@ import { resolveRouteIdFromBaseUrl, } from '../routeMetadata.js' import { resolveModelRuntimeLimits } from '../runtimeMetadata.js' -import gateway, { mapGmiCloudModel } from './gmi-cloud.js' +import { mapGmiCloudModel } from './gmi-cloud.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' -test('GMI Cloud uses a dedicated-key hybrid OpenAI-compatible gateway contract', () => { - expect(gateway.defaultBaseUrl).toBe('https://api.gmi-serving.com/v1') - expect(gateway.defaultModel).toBe('Qwen/Qwen3.8-Max') - expect(gateway.setup.credentialEnvVars).toEqual(['GMI_API_KEY']) - expect(gateway.setup.dedicatedCredentialsOnly).toBe(true) - expect(gateway.preset?.apiKeyEnvVars).toEqual(['GMI_API_KEY']) - expect(gateway.validation).toMatchObject({ - kind: 'credential-env', - routing: { matchDefaultBaseUrl: true }, - credentialEnvVars: ['GMI_API_KEY'], - }) - expect(gateway.transportConfig).toEqual({ - kind: 'openai-compatible', - openaiShim: { - supportsApiFormatSelection: false, - supportsAuthHeaders: false, - maxTokensField: 'max_tokens', +describe('gmi-cloud gateway', () => { + describe.each([ + { + routeId: 'gmi-cloud', + expected: { + baseUrl: 'https://api.gmi-serving.com/v1', + defaultModel: 'Qwen/Qwen3.8-Max', + setup: { + credentialEnvVars: ['GMI_API_KEY'], + dedicatedCredentialsOnly: true, + }, + preset: { apiKeyEnvVars: ['GMI_API_KEY'] }, + validation: { + kind: 'credential-env', + routing: { matchDefaultBaseUrl: true }, + credentialEnvVars: ['GMI_API_KEY'], + }, + openaiShim: { + supportsApiFormatSelection: false, + supportsAuthHeaders: false, + maxTokensField: 'max_tokens', + }, + catalogSource: 'hybrid', + discovery: { requiresAuth: true }, + } satisfies GatewayCatalogExpectations, }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + }) }) - expect(gateway.catalog?.source).toBe('hybrid') - expect(gateway.catalog?.discovery?.requiresAuth).toBe(true) - expect(gateway.catalog?.models?.map(model => model.apiName)).toEqual([ - 'Qwen/Qwen3.8-Max', - 'Qwen/Qwen3.8-Flash', - 'deepseek-ai/DeepSeek-V4-Pro', - 'deepseek-ai/DeepSeek-V4-Flash', - ]) }) test('GMI Cloud preset resolves to its gateway route and reads GMI env vars', () => { diff --git a/src/integrations/gateways/llmtr.test.ts b/src/integrations/gateways/llmtr.test.ts index d41fb11b1..5e53317dd 100644 --- a/src/integrations/gateways/llmtr.test.ts +++ b/src/integrations/gateways/llmtr.test.ts @@ -1,5 +1,6 @@ -import { expect, test } from 'bun:test' +import { describe, expect, test } from 'bun:test' +import { getCatalogForGateway } from '../index.js' import { routeForPreset } from '../compatibility.js' import { getProviderPresetUiMetadata } from '../providerUiMetadata.js' import { @@ -9,39 +10,45 @@ import { } from '../routeMetadata.js' import catalog, { mapLlmtrModel } from './llmtr.models.js' import gateway from './llmtr.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' -test('LLMTR uses the standard OpenAI-compatible gateway contract', () => { - expect(gateway.defaultBaseUrl).toBe('https://llmtr.com/v1') - expect(gateway.defaultModel).toBe('deepseek/deepseek-v4-flash') - expect(gateway.setup.credentialEnvVars).toEqual([ - 'LLMTR_API_KEY', - 'OPENAI_API_KEY', - ]) - expect(gateway.setup.dedicatedCredentialsOnly).not.toBe(true) - expect(gateway.preset?.apiKeyEnvVars).toEqual([ - 'LLMTR_API_KEY', - 'OPENAI_API_KEY', - ]) - expect(gateway.validation).toMatchObject({ - kind: 'credential-env', - routing: { matchDefaultBaseUrl: true }, - credentialEnvVars: [ - 'LLMTR_API_KEY', - 'OPENAI_API_KEYS', - 'OPENAI_API_KEY', - ], - }) - expect(gateway.validation?.routing?.matchBaseUrlHosts).toBeUndefined() - expect(gateway.transportConfig).toEqual({ - kind: 'openai-compatible', - openaiShim: { - requiredApiFormat: 'chat_completions', - supportsAuthHeaders: false, - maxTokensField: 'max_tokens', +describe('llmtr gateway', () => { + describe.each([ + { + routeId: 'llmtr', + expected: { + baseUrl: 'https://llmtr.com/v1', + defaultModel: 'deepseek/deepseek-v4-flash', + setup: { credentialEnvVars: ['LLMTR_API_KEY', 'OPENAI_API_KEY'] }, + preset: { apiKeyEnvVars: ['LLMTR_API_KEY', 'OPENAI_API_KEY'] }, + validation: { + kind: 'credential-env', + routing: { matchDefaultBaseUrl: true, matchBaseUrlHosts: undefined }, + credentialEnvVars: [ + 'LLMTR_API_KEY', + 'OPENAI_API_KEYS', + 'OPENAI_API_KEY', + ], + }, + openaiShim: { + requiredApiFormat: 'chat_completions', + supportsAuthHeaders: false, + maxTokensField: 'max_tokens', + }, + catalogSource: 'hybrid', + discovery: { requiresAuth: false }, + } satisfies GatewayCatalogExpectations, }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + expect(gateway.setup.dedicatedCredentialsOnly).not.toBe(true) + expect(getCatalogForGateway(routeId)).toBe(catalog) + }) }) - expect(catalog.source).toBe('hybrid') - expect(catalog.discovery?.requiresAuth).toBe(false) }) test('LLMTR preset uses the existing generic profile path', () => { @@ -174,14 +181,3 @@ test.each([ }), ).toBeNull() }) - -test('LLMTR curated models are a small tool-capable fallback catalog', () => { - expect(catalog.models?.map(model => model.apiName)).toEqual([ - 'deepseek/deepseek-v4-flash', - 'anthropic/claude-sonnet-4.6', - 'openai/gpt-5.4', - 'google/gemini-3.1-pro-preview', - 'deepseek/deepseek-v4-pro', - 'zai/glm-5.2', - ]) -}) diff --git a/src/integrations/gateways/merge-gateway.test.ts b/src/integrations/gateways/merge-gateway.test.ts index c0858637a..34ea58ddb 100644 --- a/src/integrations/gateways/merge-gateway.test.ts +++ b/src/integrations/gateways/merge-gateway.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path' import { describe, expect, mock, test } from 'bun:test' -import '../index.js' +import { getCatalogForGateway } from '../index.js' import { getRouteCredentialEnvVars, resolveActiveRouteIdFromEnv, @@ -18,6 +18,10 @@ import { } from '../../test/sharedMutationLock.js' import { asMockFetch } from '../../test/typedMocks.js' import mergeGateway from './merge-gateway.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' const mapModel = mergeGateway.catalog?.discovery?.mapModel type OpenAIShimClient = { @@ -29,28 +33,30 @@ type OpenAIShimClient = { } describe('Merge Gateway', () => { - test('uses a dedicated OpenAI-compatible gateway route', () => { - expect(mergeGateway.id).toBe('merge-gateway') - expect(mergeGateway.defaultBaseUrl).toBe( - 'https://api-gateway.merge.dev/v1/openai', - ) - expect(mergeGateway.setup.credentialEnvVars).toEqual([ - 'MERGE_GATEWAY_API_KEY', - ]) - expect(mergeGateway.setup.dedicatedCredentialsOnly).toBe(true) - expect(mergeGateway.transportConfig.kind).toBe('openai-compatible') - expect( - mergeGateway.transportConfig.openaiShim?.supportsApiFormatSelection, - ).toBe(true) - }) - - test('keeps routing policy and concrete model fallbacks in the catalog', () => { - expect(mergeGateway.catalog?.source).toBe('hybrid') - expect(mergeGateway.catalog?.discovery?.requiresAuth).toBe(true) - expect(mergeGateway.catalog?.models?.map(model => model.apiName)).toEqual([ - 'default_routing', - 'openai/gpt-5.5', - ]) + describe.each([ + { + routeId: 'merge-gateway', + expected: { + baseUrl: 'https://api-gateway.merge.dev/v1/openai', + defaultModel: 'openai/gpt-5.5', + setup: { + credentialEnvVars: ['MERGE_GATEWAY_API_KEY'], + dedicatedCredentialsOnly: true, + }, + openaiShim: { supportsApiFormatSelection: true }, + catalogSource: 'hybrid', + discovery: { requiresAuth: true }, + } satisfies GatewayCatalogExpectations, + }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + expect( + getCatalogForGateway(routeId)?.models?.some( + model => model.apiName === 'default_routing', + ), + ).toBe(true) + }) }) test('--provider selects the route and uses only its dedicated credential', async () => { diff --git a/src/integrations/gateways/nvidia-nim.test.ts b/src/integrations/gateways/nvidia-nim.test.ts index bb7b39629..96d5a2a3d 100644 --- a/src/integrations/gateways/nvidia-nim.test.ts +++ b/src/integrations/gateways/nvidia-nim.test.ts @@ -8,6 +8,10 @@ import { describe, expect, test } from 'bun:test' import nvidiaNim from './nvidia-nim.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' const mapModel = nvidiaNim.catalog?.discovery?.mapModel @@ -16,13 +20,20 @@ function shape(id: string, extras: Record = {}) { } describe('nvidia-nim gateway mapModel filter', () => { - test('mapModel is wired and gateway uses hybrid catalog', () => { - expect(mapModel).toBeDefined() - expect(nvidiaNim.catalog?.source).toBe('hybrid') - expect(nvidiaNim.catalog?.discovery?.kind).toBe('openai-compatible') - expect(nvidiaNim.catalog?.discoveryCacheTtl).toBe('1d') - expect(nvidiaNim.catalog?.discoveryRefreshMode).toBe('background-if-stale') - expect(nvidiaNim.catalog?.allowManualRefresh).toBe(true) + describe.each([ + { + routeId: 'nvidia-nim', + expected: { + catalogSource: 'hybrid', + discoveryRefreshMode: 'background-if-stale', + allowManualRefresh: true, + } satisfies GatewayCatalogExpectations, + }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + expect(nvidiaNim.catalog?.discoveryCacheTtl).toBeDefined() + }) }) test('keeps chat / instruct / reasoning / code models', () => { diff --git a/src/integrations/gateways/opencode.test.ts b/src/integrations/gateways/opencode.test.ts index aecf6829a..e3aa08913 100644 --- a/src/integrations/gateways/opencode.test.ts +++ b/src/integrations/gateways/opencode.test.ts @@ -12,6 +12,10 @@ import { acquireSharedMutationLock, releaseSharedMutationLock, } from '../../test/sharedMutationLock.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' beforeEach(async () => { await acquireSharedMutationLock('integrations/gateways/opencode.test.ts') @@ -28,6 +32,55 @@ afterEach(() => { } }) +describe('OpenCode gateway catalogs', () => { + describe.each([ + { + routeId: 'opencode', + expected: { + baseUrl: 'https://opencode.ai/zen/v1', + defaultModel: 'gpt-5.4', + category: 'aggregating', + setup: { + requiresAuth: true, + authMode: 'api-key', + credentialEnvVars: ['OPENCODE_API_KEY'], + }, + preset: { + id: 'opencode', + vendorId: 'openai', + apiKeyEnvVars: ['OPENCODE_API_KEY'], + }, + validation: { kind: 'credential-env' }, + usage: { supported: false }, + catalogSource: 'static', + } satisfies GatewayCatalogExpectations, + }, + { + routeId: 'opencode-go', + expected: { + baseUrl: 'https://opencode.ai/zen/go/v1', + defaultModel: 'glm-5.1', + category: 'aggregating', + setup: { + requiresAuth: true, + authMode: 'api-key', + credentialEnvVars: ['OPENCODE_API_KEY'], + }, + preset: { + id: 'opencode-go', + vendorId: 'openai', + apiKeyEnvVars: ['OPENCODE_API_KEY'], + }, + catalogSource: 'static', + } satisfies GatewayCatalogExpectations, + }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + }) + }) +}) + // --------------------------------------------------------------------------- // Zen Gateway Descriptor Tests // --------------------------------------------------------------------------- @@ -39,77 +92,11 @@ describe('OpenCode Zen gateway descriptor', () => { expect(gateway!.id).toBe('opencode') }) - test('has correct label', () => { - const gateway = getGateway('opencode') - expect(gateway!.label).toBe('OpenCode Zen') - }) - - test('has aggregating category', () => { - const gateway = getGateway('opencode') - expect(gateway!.category).toBe('aggregating') - }) - - test('has correct default base URL', () => { - const gateway = getGateway('opencode') - expect(gateway!.defaultBaseUrl).toBe('https://opencode.ai/zen/v1') - }) - - test('has correct default model', () => { - const gateway = getGateway('opencode') - expect(gateway!.defaultModel).toBe('gpt-5.4') - }) - - test('requires auth', () => { - const gateway = getGateway('opencode') - expect(gateway!.setup.requiresAuth).toBe(true) - }) - - test('uses api-key auth mode', () => { - const gateway = getGateway('opencode') - expect(gateway!.setup.authMode).toBe('api-key') - }) - - test('has OPENCODE_API_KEY in credential env vars', () => { - const gateway = getGateway('opencode') - expect(gateway!.setup.credentialEnvVars).toContain('OPENCODE_API_KEY') - }) - - test('has openai-compatible transport kind', () => { - const gateway = getGateway('opencode') - expect(gateway!.transportConfig.kind).toBe('openai-compatible') - }) - - test('has preset metadata', () => { - const gateway = getGateway('opencode') - expect(gateway!.preset).toBeDefined() - expect(gateway!.preset!.id).toBe('opencode') - expect(gateway!.preset!.vendorId).toBe('openai') - expect(gateway!.preset!.apiKeyEnvVars).toContain('OPENCODE_API_KEY') - }) - - test('has validation metadata', () => { - const gateway = getGateway('opencode') - expect(gateway!.validation).toBeDefined() - expect(gateway!.validation!.kind).toBe('credential-env') - }) - - test('has catalog with static source', () => { - const gateway = getGateway('opencode') - expect(gateway!.catalog).toBeDefined() - expect(gateway!.catalog!.source).toBe('static') - }) - test('has static models in catalog', () => { const gateway = getGateway('opencode') expect(gateway!.catalog!.models).toBeDefined() expect(gateway!.catalog!.models!.length).toBeGreaterThan(0) }) - - test('has usage metadata', () => { - const gateway = getGateway('opencode') - expect(gateway!.usage).toBeDefined() - expect(gateway!.usage!.supported).toBe(false) - }) }) // --------------------------------------------------------------------------- @@ -123,60 +110,6 @@ describe('OpenCode Go gateway descriptor', () => { expect(gateway!.id).toBe('opencode-go') }) - test('has correct label', () => { - const gateway = getGateway('opencode-go') - expect(gateway!.label).toBe('OpenCode Go') - }) - - test('has aggregating category', () => { - const gateway = getGateway('opencode-go') - expect(gateway!.category).toBe('aggregating') - }) - - test('has correct default base URL', () => { - const gateway = getGateway('opencode-go') - expect(gateway!.defaultBaseUrl).toBe('https://opencode.ai/zen/go/v1') - }) - - test('has correct default model', () => { - const gateway = getGateway('opencode-go') - expect(gateway!.defaultModel).toBe('glm-5.1') - }) - - test('requires auth', () => { - const gateway = getGateway('opencode-go') - expect(gateway!.setup.requiresAuth).toBe(true) - }) - - test('uses api-key auth mode', () => { - const gateway = getGateway('opencode-go') - expect(gateway!.setup.authMode).toBe('api-key') - }) - - test('has OPENCODE_API_KEY in credential env vars', () => { - const gateway = getGateway('opencode-go') - expect(gateway!.setup.credentialEnvVars).toContain('OPENCODE_API_KEY') - }) - - test('has openai-compatible transport kind', () => { - const gateway = getGateway('opencode-go') - expect(gateway!.transportConfig.kind).toBe('openai-compatible') - }) - - test('has preset metadata with vendorId openai', () => { - const gateway = getGateway('opencode-go') - expect(gateway!.preset).toBeDefined() - expect(gateway!.preset!.id).toBe('opencode-go') - expect(gateway!.preset!.vendorId).toBe('openai') - expect(gateway!.preset!.apiKeyEnvVars).toContain('OPENCODE_API_KEY') - }) - - test('has catalog with static source', () => { - const gateway = getGateway('opencode-go') - expect(gateway!.catalog).toBeDefined() - expect(gateway!.catalog!.source).toBe('static') - }) - test('has static models in catalog', () => { const gateway = getGateway('opencode-go') expect(gateway!.catalog!.models).toBeDefined() @@ -282,57 +215,6 @@ describe('OpenCode model catalog', () => { } }) - test('zen model count matches expected', () => { - const models = getCatalogEntriesForRoute('opencode') - expect(models.length).toBe(48) - }) - - test('go model count matches expected', () => { - const models = getCatalogEntriesForRoute('opencode-go') - expect(models.length).toBe(13) - }) - - test('go model set matches opencode.ai/go catalog exactly', () => { - const models = getCatalogEntriesForRoute('opencode-go') - const expectedIds = new Set([ - 'opencode-go-glm-5.2', - 'opencode-go-qwen3.7-max', - 'opencode-go-kimi-k2.7-code', - 'opencode-go-mimo-v2.5-pro', - 'opencode-go-deepseek-v4-pro', - 'opencode-go-qwen3.7-plus', - 'opencode-go-minimax-m3', - 'opencode-go-mimo-v2.5', - 'opencode-go-deepseek-v4-flash', - 'opencode-go-glm-5.1', - 'opencode-go-kimi-k2.6', - 'opencode-go-qwen3.6-plus', - 'opencode-go-minimax-m2.7', - ]) - const actualIds = new Set(models.map(m => m.id)) - // All expected IDs present - for (const id of expectedIds) { - expect(actualIds.has(id)).toBe(true) - } - // No removed or unexpected IDs remain - for (const id of actualIds) { - expect(expectedIds.has(id)).toBe(true) - } - // Explicitly verify removed models are not included - const removedIds = [ - 'opencode-go-minimax-m2.5', - 'opencode-go-kimi-k2.5', - 'opencode-go-glm-5', - 'opencode-go-qwen3.5-plus', - 'opencode-go-mimo-v2-pro', - 'opencode-go-mimo-v2-omni', - 'opencode-go-hy3-preview', - ]; - for (const id of removedIds) { - expect(actualIds.has(id)).toBe(false); - } - }) - test('all zen gpt models have modelDescriptorId', () => { const models = getCatalogEntriesForRoute('opencode') const gptModels = models.filter(m => m.apiName.startsWith('gpt-')) @@ -359,6 +241,23 @@ describe('OpenCode model catalog', () => { } }) + test('go catalog excludes removed model ids', () => { + const models = getCatalogEntriesForRoute('opencode-go') + const ids = models.map(m => m.id) + const removedModelIds = [ + 'opencode-go-minimax-m2.5', + 'opencode-go-kimi-k2.5', + 'opencode-go-glm-5', + 'opencode-go-qwen3.5-plus', + 'opencode-go-mimo-v2-pro', + 'opencode-go-mimo-v2-omni', + 'opencode-go-hy3-preview', + ] + for (const id of removedModelIds) { + expect(ids).not.toContain(id) + } + }) + test('go Anthropic-format models use the messages endpoint with x-api-key auth', () => { const models = getCatalogEntriesForRoute('opencode-go') const messagesModelIds = [ diff --git a/src/integrations/gateways/openrouter.test.ts b/src/integrations/gateways/openrouter.test.ts index 70e6a9ee9..3bf2ffb12 100644 --- a/src/integrations/gateways/openrouter.test.ts +++ b/src/integrations/gateways/openrouter.test.ts @@ -1,16 +1,23 @@ import { describe, expect, test } from 'bun:test' -import openrouter, { mapOpenRouterModel } from './openrouter.js' +import { mapOpenRouterModel } from './openrouter.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' describe('openrouter gateway live model mapping', () => { - test('uses hybrid discovery against the public models list', () => { - expect(openrouter.catalog?.source).toBe('hybrid') - expect(openrouter.catalog?.discovery).toEqual( - expect.objectContaining({ - kind: 'openai-compatible', - requiresAuth: false, - }), - ) - expect(openrouter.catalog?.discovery?.mapModel).toBe(mapOpenRouterModel) + describe.each([ + { + routeId: 'openrouter', + expected: { + catalogSource: 'hybrid', + discovery: { requiresAuth: false, mapModel: mapOpenRouterModel }, + } satisfies GatewayCatalogExpectations, + }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + }) }) test('maps coding models with context length and free labels', () => { diff --git a/src/integrations/gateways/xiaomi-mimo-token.test.ts b/src/integrations/gateways/xiaomi-mimo-token.test.ts index 385f6960a..5772021ba 100644 --- a/src/integrations/gateways/xiaomi-mimo-token.test.ts +++ b/src/integrations/gateways/xiaomi-mimo-token.test.ts @@ -6,24 +6,38 @@ import { getRouteDefaultModel, resolveRouteIdFromBaseUrl, } from '../routeMetadata.js' +import { + expectGatewayCatalog, + type GatewayCatalogExpectations, +} from '../../test/gatewayCatalogTestHelpers.js' describe('xiaomi-mimo-token gateway', () => { - test('default base URL is the token-plan SGP endpoint', () => { - expect(getRouteDefaultBaseUrl('xiaomi-mimo-token')).toBe( - 'https://token-plan-sgp.xiaomimimo.com/v1', - ) + describe.each([ + { + routeId: 'xiaomi-mimo-token', + expected: { + baseUrl: 'https://token-plan-sgp.xiaomimimo.com/v1', + defaultModel: 'mimo-v2.5-pro', + setup: { credentialEnvVars: ['MIMO_API_KEY'] }, + preset: { apiKeyEnvVars: ['MIMO_API_KEY'] }, + validation: { + kind: 'credential-env', + credentialEnvVars: ['MIMO_API_KEY', 'OPENAI_API_KEY'], + }, + catalogSource: 'static', + } satisfies GatewayCatalogExpectations, + }, + ])('$routeId catalog', ({ routeId, expected }) => { + test('satisfies the registered gateway catalog contract', () => { + expectGatewayCatalog(routeId, expected) + expect(getRouteDefaultBaseUrl(routeId)).toBe(expected.baseUrl) + expect(getRouteDefaultModel(routeId)).toBe(expected.defaultModel) + }) }) - test('default model is mimo-v2.5-pro', () => { - expect(getRouteDefaultModel('xiaomi-mimo-token')).toBe('mimo-v2.5-pro') - }) - - test('catalog is limited to supported v2.5 chat models with OpenAI-compatible reasoning effort', () => { + test('catalog models use OpenAI-compatible reasoning effort', () => { const entries = getCatalogEntriesForRoute('xiaomi-mimo-token') - expect(entries.map(model => model.apiName)).toEqual([ - 'mimo-v2.5-pro', - 'mimo-v2.5', - ]) + expect(entries.length).toBeGreaterThan(0) for (const entry of entries) { expect(entry.reasoning).toEqual({ mode: 'levels', diff --git a/src/test/gatewayCatalogTestHelpers.ts b/src/test/gatewayCatalogTestHelpers.ts new file mode 100644 index 000000000..d30ca8cac --- /dev/null +++ b/src/test/gatewayCatalogTestHelpers.ts @@ -0,0 +1,132 @@ +import { expect } from 'bun:test' + +import type { + GatewayDescriptor, + ModelCatalogConfig, + ModelDiscoveryConfig, + OpenAIShimTransportConfig, + ValidationMetadata, + ValidationRoutingMetadata, +} from '../integrations/descriptors.js' +import { parseDurationString } from '../integrations/discoveryCache.js' +import { + ensureIntegrationsLoaded, + getCatalogForGateway, + getGateway, + getModel, +} from '../integrations/index.js' + +type CredentialValidation = Extract + +export type GatewayCatalogExpectations = { + baseUrl?: GatewayDescriptor['defaultBaseUrl'] + defaultModel?: GatewayDescriptor['defaultModel'] + category?: GatewayDescriptor['category'] + supportsModelRouting?: GatewayDescriptor['supportsModelRouting'] + setup?: Partial + preset?: Partial> + validation?: Omit, 'routing'> & { + routing?: Partial + } + openaiShim?: Partial + discovery?: Partial + catalogSource: ModelCatalogConfig['source'] + discoveryRefreshMode?: ModelCatalogConfig['discoveryRefreshMode'] + allowManualRefresh?: ModelCatalogConfig['allowManualRefresh'] + usage?: Partial> + allowEmptyCatalog?: boolean +} + +// Iterating supplied keys preserves explicit undefined expectations. +function expectFields( + actual: T | undefined, + expected: Partial, +): void { + expect(actual).toBeDefined() + if (!actual) throw new Error('Expected gateway metadata to exist') + for (const key of Object.keys(expected) as (keyof T)[]) { + expect(actual[key]).toEqual(expected[key]) + } +} + +export function expectGatewayCatalog( + routeId: string, + expected: GatewayCatalogExpectations, +): void { + ensureIntegrationsLoaded() + const gateway = getGateway(routeId) + const catalog = getCatalogForGateway(routeId) + expect(gateway).toBeDefined() + expect(catalog).toBeDefined() + if (!gateway || !catalog) throw new Error(`Missing gateway catalog: ${routeId}`) + + expect(gateway.id).toBe(routeId) + expect(gateway.label.trim().length).toBeGreaterThan(0) + if ('baseUrl' in expected) expect(gateway.defaultBaseUrl).toBe(expected.baseUrl) + if ('defaultModel' in expected) { + expect(gateway.defaultModel).toBe(expected.defaultModel) + } + if ('category' in expected) expect(gateway.category).toBe(expected.category) + if ('supportsModelRouting' in expected) { + expect(gateway.supportsModelRouting).toBe(expected.supportsModelRouting) + } + if (expected.setup) expectFields(gateway.setup, expected.setup) + if (expected.preset) expectFields(gateway.preset, expected.preset) + if (expected.validation) { + const { routing, ...validation } = expected.validation + expectFields(gateway.validation, validation) + if (routing) expectFields(gateway.validation?.routing, routing) + } + if (expected.usage) expectFields(gateway.usage, expected.usage) + expect(gateway.transportConfig.kind).toBe('openai-compatible') + if (expected.openaiShim) { + expectFields(gateway.transportConfig.openaiShim, expected.openaiShim) + } + + expect(catalog.source).toBe(expected.catalogSource) + if (catalog.source === 'dynamic' || catalog.source === 'hybrid') { + expect(catalog.discovery?.kind).toBe('openai-compatible') + expect(typeof catalog.discovery?.mapModel).toBe('function') + } + if (expected.discovery) { + const { mapModel, ...discovery } = expected.discovery + expectFields(catalog.discovery, discovery) + if ('mapModel' in expected.discovery) { + expect(catalog.discovery?.mapModel).toBe(mapModel) + } + } + if (catalog.discoveryCacheTtl !== undefined) { + const ttl = parseDurationString(catalog.discoveryCacheTtl) + expect(Number.isFinite(ttl)).toBe(true) + expect(ttl).toBeGreaterThan(0) + } + if ('discoveryRefreshMode' in expected) { + expect(catalog.discoveryRefreshMode).toBe(expected.discoveryRefreshMode) + } + if ('allowManualRefresh' in expected) { + expect(catalog.allowManualRefresh).toBe(expected.allowManualRefresh) + } + + // Raw entries include hidden and expired models filtered out by route lookup. + const entries = catalog.models ?? [] + if (!expected.allowEmptyCatalog) { + expect(entries.length).toBeGreaterThan(0) + if (gateway.defaultModel !== undefined) { + expect( + entries.some(entry => entry.apiName === gateway.defaultModel), + ).toBe(true) + } + } + expect(new Set(entries.map(entry => entry.id)).size).toBe(entries.length) + expect(new Set(entries.map(entry => entry.apiName)).size).toBe(entries.length) + for (const entry of entries) { + expect(entry.id.trim().length).toBeGreaterThan(0) + expect(entry.apiName.trim().length).toBeGreaterThan(0) + if (entry.label !== undefined) { + expect(entry.label.trim().length).toBeGreaterThan(0) + } + if (entry.modelDescriptorId !== undefined) { + expect(getModel(entry.modelDescriptorId)).toBeDefined() + } + } +}