From c84411b9dd783a6154838537a0561705c70dc0ab Mon Sep 17 00:00:00 2001 From: JATMN Date: Thu, 22 Oct 2026 05:46:03 -0700 Subject: [PATCH] test: run attribution suites in a child when auth, beta or routing exports are polluted (TA-001 slice 5) (#2334) --- .../api/authRouting.attribution.test.ts | 342 +++++----- .../providerModuleIsolation.fixture.ts | 107 +++ src/test/providerModuleIsolation.test.ts | 107 +++ src/test/providerModuleIsolation.ts | 56 ++ src/utils/sideQuery.attribution.test.ts | 643 +++++++++--------- 5 files changed, 745 insertions(+), 510 deletions(-) create mode 100644 src/test/fixtures/providerModuleIsolation.fixture.ts create mode 100644 src/test/providerModuleIsolation.test.ts diff --git a/src/services/api/authRouting.attribution.test.ts b/src/services/api/authRouting.attribution.test.ts index 4dba7b97d..c38ac77be 100644 --- a/src/services/api/authRouting.attribution.test.ts +++ b/src/services/api/authRouting.attribution.test.ts @@ -1,182 +1,185 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test' -import { acquireSharedMutationLock, releaseSharedMutationLock } from '../../test/sharedMutationLock.js' -import { applyAnthropicAttributionPolicy } from '../../utils/anthropicAttribution.js' import { - providerModuleIsMocked, - REAL_PROVIDER_TEST_CHILD_ENV, + attributionModulesNeedIsolation, REAL_PROVIDER_TEST_TIMEOUT_MS, runTestFileWithRealProviders, } from '../../test/providerModuleIsolation.js' -// Bun keeps mock.module() registrations process-global across test files. -// Bind request modules only when the canonical provider module is real. When a -// prior file replaced it, run this file in a clean child process instead. -const _realProvidersModule = await import( - `../../utils/model/providers.js?attributionReal=${Date.now()}-${Math.random()}` -) -const _loadedProvidersModule = await import('src/utils/model/providers.js') -const runInProviderIsolatedChild = - process.env[REAL_PROVIDER_TEST_CHILD_ENV] !== '1' && - providerModuleIsMocked(_loadedProvidersModule, _realProvidersModule) -type AuthRoutingModule = typeof import('./authRouting.js') -let resolveAnthropicAttributionAuthFromSources!: AuthRoutingModule['resolveAnthropicAttributionAuthFromSources'] -let resolveCurrentAnthropicAttributionPolicy!: AuthRoutingModule['resolveCurrentAnthropicAttributionPolicy'] -if (!runInProviderIsolatedChild) { - ;({ - resolveAnthropicAttributionAuthFromSources, - resolveCurrentAnthropicAttributionPolicy, - } = await import( - `./authRouting.js?attributionReal=${Date.now()}-${Math.random()}` - )) -} - -const routeEnvKeys = [ - 'ANTHROPIC_API_KEY', - 'ANTHROPIC_AUTH_TOKEN', - 'ANTHROPIC_BASE_URL', - 'ANTHROPIC_MODEL', - 'CLAUDE_CODE_ENTRYPOINT', - 'CLAUDE_CODE_OAUTH_TOKEN', - 'CLAUDE_CODE_REMOTE', - 'CLAUDE_CODE_USE_BEDROCK', - 'CLAUDE_CODE_USE_FOUNDRY', - 'CLAUDE_CODE_USE_GEMINI', - 'CLAUDE_CODE_USE_GITHUB', - 'CLAUDE_CODE_USE_MISTRAL', - 'CLAUDE_CODE_USE_OPENAI', - 'CLAUDE_CODE_USE_VERTEX', - 'MINIMAX_API_KEY', - 'OPENAI_BASE_URL', - 'OPENAI_MODEL', -] as const -const originalEnv = { ...process.env } - -beforeEach(async () => { - await acquireSharedMutationLock('authRouting.attribution.test.ts') - for (const key of routeEnvKeys) delete process.env[key] - process.env.ANTHROPIC_API_KEY = 'sk-test-attribution-routing' -}) - -afterEach(() => { - try { - for (const key of routeEnvKeys) { - if (originalEnv[key] === undefined) { - delete process.env[key] - } else { - process.env[key] = originalEnv[key] - } - } - } finally { - releaseSharedMutationLock() - } -}) - -if (runInProviderIsolatedChild) { +// Inspect canonical dependencies before binding the subject or suite setup. +const needsChild = await attributionModulesNeedIsolation() +if (needsChild) { test('runs attribution route cases with the real provider module', async () => { await runTestFileWithRealProviders(import.meta.path) }, { timeout: REAL_PROVIDER_TEST_TIMEOUT_MS + 5_000 }) -} +} else { + const { acquireSharedMutationLock, releaseSharedMutationLock } = + await import('../../test/sharedMutationLock.js') + const { applyAnthropicAttributionPolicy } = + await import('../../utils/anthropicAttribution.js') + const { + resolveAnthropicAttributionAuthFromSources, + resolveCurrentAnthropicAttributionPolicy, + }: typeof import('./authRouting.js') = await import( + `./authRouting.js?attributionReal=${Date.now()}-${Math.random()}` + ) -const describeAttribution = runInProviderIsolatedChild - ? describe.skip - : describe - -describeAttribution('current Anthropic attribution route resolution', () => { - for (const apiKeySource of [ + const routeEnvKeys = [ 'ANTHROPIC_API_KEY', - 'apiKeyHelper', - ] as const) { - test(`ignores ${apiKeySource} when managed OAuth is effective`, () => { - expect( - resolveAnthropicAttributionAuthFromSources({ - apiKeySource, - authTokenSource: 'CLAUDE_CODE_OAUTH_TOKEN', - bareMode: false, - isSubscriber: true, - managedOAuthContext: true, - }), - ).toBe('oauth_subscription') - }) - } + 'ANTHROPIC_AUTH_TOKEN', + 'ANTHROPIC_BASE_URL', + 'ANTHROPIC_MODEL', + 'CLAUDE_CODE_ENTRYPOINT', + 'CLAUDE_CODE_OAUTH_TOKEN', + 'CLAUDE_CODE_REMOTE', + 'CLAUDE_CODE_USE_BEDROCK', + 'CLAUDE_CODE_USE_FOUNDRY', + 'CLAUDE_CODE_USE_GEMINI', + 'CLAUDE_CODE_USE_GITHUB', + 'CLAUDE_CODE_USE_MISTRAL', + 'CLAUDE_CODE_USE_OPENAI', + 'CLAUDE_CODE_USE_VERTEX', + 'MINIMAX_API_KEY', + 'OPENAI_BASE_URL', + 'OPENAI_MODEL', + ] as const + const originalEnv = { ...process.env } - for (const [apiKeySource, authTokenSource] of [ - ['ANTHROPIC_API_KEY', 'none'], - ['apiKeyHelper', 'apiKeyHelper'], - ] as const) { - test(`preserves bare-mode ${apiKeySource} in a managed context`, () => { + beforeEach(async () => { + await acquireSharedMutationLock('authRouting.attribution.test.ts') + for (const key of routeEnvKeys) delete process.env[key] + process.env.ANTHROPIC_API_KEY = 'sk-test-attribution-routing' + }) + + afterEach(() => { + try { + for (const key of routeEnvKeys) { + if (originalEnv[key] === undefined) { + delete process.env[key] + } else { + process.env[key] = originalEnv[key] + } + } + } finally { + releaseSharedMutationLock() + } + }) + + describe('current Anthropic attribution route resolution', () => { + for (const apiKeySource of [ + 'ANTHROPIC_API_KEY', + 'apiKeyHelper', + ] as const) { + test(`ignores ${apiKeySource} when managed OAuth is effective`, () => { + expect( + resolveAnthropicAttributionAuthFromSources({ + apiKeySource, + authTokenSource: 'CLAUDE_CODE_OAUTH_TOKEN', + bareMode: false, + isSubscriber: true, + managedOAuthContext: true, + }), + ).toBe('oauth_subscription') + }) + } + + for (const [apiKeySource, authTokenSource] of [ + ['ANTHROPIC_API_KEY', 'none'], + ['apiKeyHelper', 'apiKeyHelper'], + ] as const) { + test(`preserves bare-mode ${apiKeySource} in a managed context`, () => { + expect( + resolveAnthropicAttributionAuthFromSources({ + apiKeySource, + authTokenSource, + bareMode: true, + isSubscriber: false, + managedOAuthContext: true, + }), + ).toBe('api_key') + }) + } + + test('preserves a managed API key when managed OAuth is not effective', () => { expect( resolveAnthropicAttributionAuthFromSources({ - apiKeySource, - authTokenSource, - bareMode: true, + apiKeySource: '/login managed key', + authTokenSource: 'none', + bareMode: false, isSubscriber: false, managedOAuthContext: true, }), ).toBe('api_key') }) - } - - test('preserves a managed API key when managed OAuth is not effective', () => { - expect( - resolveAnthropicAttributionAuthFromSources({ - apiKeySource: '/login managed key', - authTokenSource: 'none', - bareMode: false, - isSubscriber: false, - managedOAuthContext: true, - }), - ).toBe('api_key') - }) - - test('does not force managed OAuth past an explicit non-subscriber result', () => { - expect( - resolveAnthropicAttributionAuthFromSources({ - apiKeySource: 'none', - authTokenSource: 'CLAUDE_CODE_OAUTH_TOKEN', - bareMode: false, - isSubscriber: false, - managedOAuthContext: true, - }), - ).toBe('unknown') - }) - - for (const [label, envKey, envValue] of [ - ['remote', 'CLAUDE_CODE_REMOTE', '1'], - ['Claude Desktop', 'CLAUDE_CODE_ENTRYPOINT', 'claude-desktop'], - ] as const) { - test(`uses the effective OAuth credential for ${label} sessions`, () => { - process.env[envKey] = envValue - process.env.CLAUDE_CODE_OAUTH_TOKEN = 'oauth-test-token' + test('does not force managed OAuth past an explicit non-subscriber result', () => { expect( - resolveCurrentAnthropicAttributionPolicy({ - attributionEnabled: false, + resolveAnthropicAttributionAuthFromSources({ + apiKeySource: 'none', + authTokenSource: 'CLAUDE_CODE_OAUTH_TOKEN', + bareMode: false, + isSubscriber: false, + managedOAuthContext: true, }), - ).toEqual({ - generate: true, - retain: true, - reason: 'official_oauth_required', - }) + ).toBe('unknown') }) - } - for (const [label, envKey] of [ - ['Bedrock', 'CLAUDE_CODE_USE_BEDROCK'], - ['Vertex', 'CLAUDE_CODE_USE_VERTEX'], - ['Foundry', 'CLAUDE_CODE_USE_FOUNDRY'], - ] as const) { - test(`does not emit first-party metadata on ${label}`, () => { - process.env[envKey] = '1' + for (const [label, envKey, envValue] of [ + ['remote', 'CLAUDE_CODE_REMOTE', '1'], + ['Claude Desktop', 'CLAUDE_CODE_ENTRYPOINT', 'claude-desktop'], + ] as const) { + test(`uses the effective OAuth credential for ${label} sessions`, () => { + process.env[envKey] = envValue + process.env.CLAUDE_CODE_OAUTH_TOKEN = 'oauth-test-token' + expect( + resolveCurrentAnthropicAttributionPolicy({ + attributionEnabled: false, + }), + ).toEqual({ + generate: true, + retain: true, + reason: 'official_oauth_required', + }) + }) + } + + for (const [label, envKey] of [ + ['Bedrock', 'CLAUDE_CODE_USE_BEDROCK'], + ['Vertex', 'CLAUDE_CODE_USE_VERTEX'], + ['Foundry', 'CLAUDE_CODE_USE_FOUNDRY'], + ] as const) { + test(`does not emit first-party metadata on ${label}`, () => { + process.env[envKey] = '1' + + const policy = resolveCurrentAnthropicAttributionPolicy({ + attributionEnabled: true, + }) + + expect(policy).toEqual({ + generate: false, + retain: false, + reason: 'non_official_route', + }) + expect( + applyAnthropicAttributionPolicy( + ['x-anthropic-billing-header: stale', 'stable route prompt'], + policy, + ), + ).toEqual(['stable route prompt']) + }) + } + + test('does not emit through a provider override', () => { const policy = resolveCurrentAnthropicAttributionPolicy({ attributionEnabled: true, + providerOverride: { + model: 'third-party-model', + baseURL: 'https://provider.example/v1', + apiKey: 'provider-test-key', + }, }) - expect(policy).toEqual({ - generate: false, - retain: false, - reason: 'non_official_route', - }) + expect(policy).toMatchObject({ generate: false, retain: false }) expect( applyAnthropicAttributionPolicy( ['x-anthropic-billing-header: stale', 'stable route prompt'], @@ -184,35 +187,16 @@ describeAttribution('current Anthropic attribution route resolution', () => { ), ).toEqual(['stable route prompt']) }) - } - test('does not emit through a provider override', () => { - const policy = resolveCurrentAnthropicAttributionPolicy({ - attributionEnabled: true, - providerOverride: { - model: 'third-party-model', - baseURL: 'https://provider.example/v1', - apiKey: 'provider-test-key', - }, + test('treats a custom Anthropic Messages route as non-official', () => { + process.env.ANTHROPIC_BASE_URL = 'https://messages.example/v1' + process.env.ANTHROPIC_MODEL = 'claude-compatible' + + expect( + resolveCurrentAnthropicAttributionPolicy({ + attributionEnabled: true, + }), + ).toMatchObject({ generate: false, retain: false }) }) - - expect(policy).toMatchObject({ generate: false, retain: false }) - expect( - applyAnthropicAttributionPolicy( - ['x-anthropic-billing-header: stale', 'stable route prompt'], - policy, - ), - ).toEqual(['stable route prompt']) }) - - test('treats a custom Anthropic Messages route as non-official', () => { - process.env.ANTHROPIC_BASE_URL = 'https://messages.example/v1' - process.env.ANTHROPIC_MODEL = 'claude-compatible' - - expect( - resolveCurrentAnthropicAttributionPolicy({ - attributionEnabled: true, - }), - ).toMatchObject({ generate: false, retain: false }) - }) -}) +} diff --git a/src/test/fixtures/providerModuleIsolation.fixture.ts b/src/test/fixtures/providerModuleIsolation.fixture.ts new file mode 100644 index 000000000..0304ac2ff --- /dev/null +++ b/src/test/fixtures/providerModuleIsolation.fixture.ts @@ -0,0 +1,107 @@ +// Imported only by disposable test entrypoints; never discovered as a suite. +import { expect, mock, test } from 'bun:test' +import { + attributionModulesNeedIsolation, + providerModuleIsMocked, + REAL_PROVIDER_TEST_CHILD_ENV, +} from '../providerModuleIsolation.js' + +const scenario = process.env.ISOLATION_FIXTURE_SCENARIO +const inChild = process.env[REAL_PROVIDER_TEST_CHILD_ENV] === '1' + +if (scenario === 'assertion-failure') { + test('child assertion failure sentinel', () => { + expect('actual child value').toBe('expected child value') + }) +} else if (scenario?.startsWith('suite-')) { + if (!inChild) { + const auth = await import('../../utils/auth.js') + mock.module('../../utils/auth.js', () => ({ + ...auth, + isClaudeAISubscriber: () => false, + })) + // Any real-suite hook in the parent must fail, including cleanup hooks. + mock.module('../sharedMutationLock.js', () => ({ + acquireSharedMutationLock: () => { + throw new Error('parent setup sentinel') + }, + releaseSharedMutationLock: () => { + throw new Error('parent cleanup sentinel') + }, + })) + } + if (scenario === 'suite-sideQuery') { + await import('../../utils/sideQuery.attribution.test.js') + } else { + await import('../../services/api/authRouting.attribution.test.js') + } +} else { + if (scenario === 'auth') { + const auth = await import('../../utils/auth.js') + mock.module('../../utils/auth.js', () => ({ + ...auth, + isClaudeAISubscriber: () => false, + })) + // Provider restoration must not hide independent auth pollution. + const realProviders = await import( + `../../utils/model/providers.js?fixtureReal=${Math.random()}` + ) + mock.module('../../utils/model/providers.js', () => ({ ...realProviders })) + const providers = await import('../../utils/model/providers.js') + expect(providerModuleIsMocked(providers, realProviders)).toBe(false) + } else if (scenario === 'betas') { + const betas = await import('../../utils/betas.js') + mock.module('../../utils/betas.js', () => ({ + ...betas, + shouldUseGlobalCacheScope: () => false, + })) + } else if (scenario === 'routing') { + const routing = await import('../../services/api/authRouting.js') + mock.module('../../services/api/authRouting.js', () => ({ + ...routing, + shouldUseCustomAnthropicBearerAuth: () => true, + })) + } else if (scenario === 'metadata') { + const metadata = await import('../../integrations/routeMetadata.js') + mock.module('../../integrations/routeMetadata.js', () => ({ + ...metadata, + getRouteLabel: () => 'polluted route', + })) + } else if (scenario === 'client') { + const client = await import('../../services/api/client.js') + mock.module('../../services/api/client.js', () => ({ + ...client, + CLIENT_REQUEST_ID_HEADER: 'polluted-header', + })) + } else if (scenario === 'missing') { + // Register before loading: Bun retains omitted exports when replacing an + // already-loaded namespace, which would not exercise a missing surface. + mock.module('../../utils/model/providers.js', () => ({})) + const loaded = await import('../../utils/model/providers.js') + expect(Object.keys(loaded)).toHaveLength(0) + expect(loaded.isFirstPartyAnthropicBaseUrl).toBeUndefined() + } else if (scenario === 'unsupported' || scenario === 'native-function') { + const providers = await import('../../utils/model/providers.js') + mock.module('../../utils/model/providers.js', () => ({ + ...providers, + getAPIProvider: scenario === 'unsupported' + ? {} + : providers.getAPIProvider.bind(null), + })) + } else if (scenario === 'import-failure') { + mock.module('../../utils/model/providers.js', () => { + throw new Error('module inspection sentinel') + }) + } else if (scenario === 'child-marker') { + process.env[REAL_PROVIDER_TEST_CHILD_ENV] = '1' + mock.module('../../utils/model/providers.js', () => { + throw new Error('child detector imported providers') + }) + } + + test(`attribution isolation detector: ${scenario}`, async () => { + expect(await attributionModulesNeedIsolation()).toBe( + scenario !== 'clean' && scenario !== 'child-marker', + ) + }) +} diff --git a/src/test/providerModuleIsolation.test.ts b/src/test/providerModuleIsolation.test.ts new file mode 100644 index 000000000..575877879 --- /dev/null +++ b/src/test/providerModuleIsolation.test.ts @@ -0,0 +1,107 @@ +import { expect, test } from 'bun:test' +import { mkdtempSync, rmSync, writeFileSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { + REAL_PROVIDER_TEST_CHILD_ENV, + REAL_PROVIDER_TEST_TIMEOUT_MS, +} from './providerModuleIsolation.js' + +async function withFixture( + scenario: string, + run: (entry: string, env: NodeJS.ProcessEnv) => Promise, +): Promise { + const root = mkdtempSync(join(tmpdir(), 'provider-isolation-')) + const entry = join(root, 'isolation.test.ts') + const fixture = join(import.meta.dir, 'fixtures/providerModuleIsolation.fixture.ts') + writeFileSync(entry, `await import(${JSON.stringify(fixture)})\n`) + try { + await run(entry, { + ...process.env, + [REAL_PROVIDER_TEST_CHILD_ENV]: undefined, + ISOLATION_FIXTURE_SCENARIO: scenario, + CLAUDE_CONFIG_DIR: root, + OPENCLAUDE_CONFIG_DIR: root, + }) + } finally { + rmSync(root, { recursive: true, force: true }) + } +} + +// The fixture may itself run a child with a 30-second deadline. Give the outer +// process and test additional time for diagnostics and cleanup. +const fixtureTimeout = REAL_PROVIDER_TEST_TIMEOUT_MS + 10_000 +const testTimeout = fixtureTimeout + 5_000 + +for (const scenario of [ + 'clean', + 'auth', + 'betas', + 'routing', + 'metadata', + 'client', + 'missing', + 'unsupported', + 'native-function', + 'import-failure', + 'child-marker', + 'suite-sideQuery', + 'suite-authRouting', +]) { + test(`isolates attribution modules: ${scenario}`, async () => { + await withFixture(scenario, async (entry, env) => { + const child = Bun.spawn({ + cmd: [process.execPath, 'test', entry], + cwd: process.cwd(), + env, + stdout: 'pipe', + stderr: 'pipe', + timeout: fixtureTimeout, + killSignal: process.platform === 'win32' ? undefined : 'SIGKILL', + }) + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]) + const output = `${stdout}\n${stderr}` + expect(output).not.toContain('parent setup sentinel') + expect(output).not.toContain('parent cleanup sentinel') + expect(output).toMatch(/^\s*1 pass$/m) + expect(output).toMatch(/^\s*0 fail$/m) + expect(output).not.toMatch(/\d+ skip/) + expect(exitCode).toBe(0) + }) + }, { timeout: testTimeout }) +} + +test('propagates child assertion failures through the launcher', async () => { + await withFixture('assertion-failure', async (entry, env) => { + // Run the real launcher in a disposable process so coordinator env and mocks + // remain untouched even when other suites share this process. + const runner = join(entry, '..', 'launcher.ts') + writeFileSync(runner, ` + import { runTestFileWithRealProviders } from ${JSON.stringify( + join(import.meta.dir, 'providerModuleIsolation.ts'), + )} + await runTestFileWithRealProviders(${JSON.stringify(entry)}) + `) + const child = Bun.spawn({ + cmd: [process.execPath, runner], + env, + stdout: 'pipe', + stderr: 'pipe', + timeout: fixtureTimeout, + killSignal: process.platform === 'win32' ? undefined : 'SIGKILL', + }) + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]) + expect(exitCode).not.toBe(0) + expect(`${stdout}\n${stderr}`).toContain('provider-isolated test process failed') + expect(`${stdout}\n${stderr}`).toContain('child assertion failure sentinel') + expect(`${stdout}\n${stderr}`).toContain('expected child value') + }) +}, { timeout: testTimeout }) diff --git a/src/test/providerModuleIsolation.ts b/src/test/providerModuleIsolation.ts index e7a329a18..938812350 100644 --- a/src/test/providerModuleIsolation.ts +++ b/src/test/providerModuleIsolation.ts @@ -32,6 +32,62 @@ export function providerModuleIsMocked( ) } +function exportSurfacesDiffer(loaded: object, real: object): boolean { + const loadedKeys = Reflect.ownKeys(loaded) + const realKeys = Reflect.ownKeys(real) + if ( + loadedKeys.length !== realKeys.length || + realKeys.some(key => !Object.hasOwn(loaded, key)) + ) return true + + return realKeys.some(key => { + const actual: unknown = Reflect.get(loaded, key) + const expected: unknown = Reflect.get(real, key) + if (typeof actual !== typeof expected) return true + if (typeof actual === 'function' && typeof expected === 'function') { + const actualSource = Function.prototype.toString.call(actual) + const expectedSource = Function.prototype.toString.call(expected) + // Native/bound functions cannot be inspected reliably. + return actualSource.includes('[native code]') || + expectedSource.includes('[native code]') || actualSource !== expectedSource + } + // These modules currently export functions and primitives. Unknown mutable + // objects (including symbols) cannot establish a clean comparison. + if ((typeof actual === 'object' && actual !== null) || + typeof actual === 'symbol') return true + return !Object.is(actual, expected) + }) +} + +export async function attributionModulesNeedIsolation(): Promise { + // Check before any imports: a clean child must never recurse. + if (process.env[REAL_PROVIDER_TEST_CHILD_ENV] === '1') return false + + // This detects replaced exports, not arbitrary stale dependency caches. + // Auth is checked independently even when routing has unchanged source. + const modulePaths = [ + '../utils/model/providers.js', + '../utils/auth.js', + '../utils/betas.js', + '../services/api/authRouting.js', + '../integrations/routeMetadata.js', + '../services/api/client.js', + ] + try { + for (const path of modulePaths) { + const loaded = await import(path) + const real = await import( + `${path}?attributionReal=${Date.now()}-${Math.random()}` + ) + if (exportSurfacesDiffer(loaded, real)) return true + } + return false + } catch { + // Missing exports, throwing getters, and failed imports require isolation. + return true + } +} + export async function runTestFileWithRealProviders( testFile: string, ): Promise { diff --git a/src/utils/sideQuery.attribution.test.ts b/src/utils/sideQuery.attribution.test.ts index b9a0ea98b..17ca9b845 100644 --- a/src/utils/sideQuery.attribution.test.ts +++ b/src/utils/sideQuery.attribution.test.ts @@ -1,270 +1,281 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test' -import { mkdtempSync, rmSync, writeFileSync } from 'fs' -import { tmpdir } from 'os' -import { join } from 'path' import { - setAllowedSettingSources, - setFlagSettingsInline, - setFlagSettingsPath, -} from '../bootstrap/state.js' -import { resetGrowthBook } from '../services/analytics/growthbook.js' -import { acquireSharedMutationLock, releaseSharedMutationLock } from '../test/sharedMutationLock.js' -import { getClaudeAIOAuthTokens } from './auth.js' -import { enableConfigs } from './config.js' -import { SETTING_SOURCES } from './settings/constants.js' -import { resetSettingsCache } from './settings/settingsCache.js' -import { - providerModuleIsMocked, - REAL_PROVIDER_TEST_CHILD_ENV, + attributionModulesNeedIsolation, REAL_PROVIDER_TEST_TIMEOUT_MS, runTestFileWithRealProviders, } from '../test/providerModuleIsolation.js' -// Bun keeps mock.module() registrations process-global across test files. -// Bind sideQuery only when the canonical provider module is real. When a prior -// file replaced it, run this file in a clean child process instead. -const _realProvidersModule = await import( - `./model/providers.js?attributionReal=${Date.now()}-${Math.random()}` -) -const _loadedProvidersModule = await import('src/utils/model/providers.js') -const runInProviderIsolatedChild = - process.env[REAL_PROVIDER_TEST_CHILD_ENV] !== '1' && - providerModuleIsMocked(_loadedProvidersModule, _realProvidersModule) -type SideQueryModule = typeof import('./sideQuery.js') -let sideQuery!: SideQueryModule['sideQuery'] -if (!runInProviderIsolatedChild) { - ;({ sideQuery } = await import( +// Inspect canonical dependencies before binding the subject or suite setup. +const needsChild = await attributionModulesNeedIsolation() +if (needsChild) { + test('runs side-query attribution cases with the real provider module', async () => { + await runTestFileWithRealProviders(import.meta.path) + }, { timeout: REAL_PROVIDER_TEST_TIMEOUT_MS + 5_000 }) +} else { + const { mkdtempSync, rmSync, writeFileSync } = await import('fs') + const { tmpdir } = await import('os') + const { join } = await import('path') + const { setAllowedSettingSources, setFlagSettingsInline, setFlagSettingsPath } = + await import('../bootstrap/state.js') + const { resetGrowthBook } = await import('../services/analytics/growthbook.js') + const { acquireSharedMutationLock, releaseSharedMutationLock } = + await import('../test/sharedMutationLock.js') + const { getClaudeAIOAuthTokens } = await import('./auth.js') + const { enableConfigs } = await import('./config.js') + const { SETTING_SOURCES } = await import('./settings/constants.js') + const { resetSettingsCache } = await import('./settings/settingsCache.js') + const { sideQuery }: typeof import('./sideQuery.js') = await import( `./sideQuery.js?attributionReal=${Date.now()}-${Math.random()}` - )) -} + ) -const envKeys = [ - 'ANTHROPIC_API_KEY', - 'ANTHROPIC_AUTH_TOKEN', - 'ANTHROPIC_BASE_URL', - 'ANTHROPIC_MODEL', - 'ANTHROPIC_UNIX_SOCKET', - 'CLAUDE_CODE_ATTRIBUTION_HEADER', - 'CLAUDE_CODE_ENTRYPOINT', - 'CLAUDE_CODE_OAUTH_TOKEN', - 'CLAUDE_CODE_REMOTE', - 'CLAUDE_CODE_USE_BEDROCK', - 'CLAUDE_CODE_USE_FOUNDRY', - 'CLAUDE_CODE_USE_GEMINI', - 'CLAUDE_CODE_USE_GITHUB', - 'CLAUDE_CODE_USE_MISTRAL', - 'CLAUDE_CODE_USE_OPENAI', - 'CLAUDE_CODE_USE_VERTEX', - 'CLAUDE_FEATURE_FLAGS_FILE', - 'OPENAI_API_BASE', - 'OPENAI_API_KEY', - 'OPENAI_BASE_URL', - 'OPENAI_MODEL', - 'OPENCLAUDE_CONFIG_DIR', -] as const -const originalEnv = { ...process.env } -const originalFetch = globalThis.fetch -const hadSavedMacro = Object.hasOwn(globalThis, 'MACRO') -const savedMacro = (globalThis as Record).MACRO -const originalNodeEnv = process.env.NODE_ENV -let configRoot: string | undefined + const envKeys = [ + 'ANTHROPIC_API_KEY', + 'ANTHROPIC_AUTH_TOKEN', + 'ANTHROPIC_BASE_URL', + 'ANTHROPIC_MODEL', + 'ANTHROPIC_UNIX_SOCKET', + 'CLAUDE_CODE_ATTRIBUTION_HEADER', + 'CLAUDE_CODE_ENTRYPOINT', + 'CLAUDE_CODE_OAUTH_TOKEN', + 'CLAUDE_CODE_REMOTE', + 'CLAUDE_CODE_USE_BEDROCK', + 'CLAUDE_CODE_USE_FOUNDRY', + 'CLAUDE_CODE_USE_GEMINI', + 'CLAUDE_CODE_USE_GITHUB', + 'CLAUDE_CODE_USE_MISTRAL', + 'CLAUDE_CODE_USE_OPENAI', + 'CLAUDE_CODE_USE_VERTEX', + 'CLAUDE_FEATURE_FLAGS_FILE', + 'OPENAI_API_BASE', + 'OPENAI_API_KEY', + 'OPENAI_BASE_URL', + 'OPENAI_MODEL', + 'OPENCLAUDE_CONFIG_DIR', + ] as const + const originalEnv = { ...process.env } + const originalFetch = globalThis.fetch + const hadSavedMacro = Object.hasOwn(globalThis, 'MACRO') + const savedMacro = (globalThis as Record).MACRO + const originalNodeEnv = process.env.NODE_ENV + let configRoot: string | undefined -function makeMessageResponse(): Response { - return new Response( - JSON.stringify({ - id: 'msg-side-attribution-test', - type: 'message', - role: 'assistant', + function makeMessageResponse(): Response { + return new Response( + JSON.stringify({ + id: 'msg-side-attribution-test', + type: 'message', + role: 'assistant', + model: 'claude-side-attribution-test', + content: [], + container: null, + context_management: null, + stop_details: null, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 1, + output_tokens: 1, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + }), + { + headers: { + 'content-type': 'application/json', + 'request-id': 'req-side-attribution-test', + }, + }, + ) + } + + async function captureSideQueryRequest( + system: + | string + | Array<{ + type: 'text' + text: string + cache_control?: { type: 'ephemeral'; scope?: 'global' | 'org' } + }> = 'stable side prompt', + ): Promise<{ + system: Array> + headers: Headers + }> { + let requestBody: Record | undefined + let requestHeaders: Headers | undefined + globalThis.fetch = (async (input, init) => { + const request = + input instanceof Request + ? input.clone() + : new Request(input as RequestInfo, init) + const body = await request.text() + if (body) requestBody = JSON.parse(body) as Record + requestHeaders = new Headers(request.headers) + return makeMessageResponse() + }) as typeof fetch + + await sideQuery({ + querySource: 'model_validation', model: 'claude-side-attribution-test', - content: [], - container: null, - context_management: null, - stop_details: null, - stop_reason: 'end_turn', - stop_sequence: null, - usage: { - input_tokens: 1, - output_tokens: 1, - cache_creation_input_tokens: 0, - cache_read_input_tokens: 0, - }, - }), - { - headers: { - 'content-type': 'application/json', - 'request-id': 'req-side-attribution-test', - }, - }, - ) -} + system, + messages: [{ role: 'user', content: 'hello' }], + }) -async function captureSideQueryRequest( - system: - | string - | Array<{ - type: 'text' - text: string - cache_control?: { type: 'ephemeral'; scope?: 'global' | 'org' } - }> = 'stable side prompt', -): Promise<{ - system: Array> - headers: Headers -}> { - let requestBody: Record | undefined - let requestHeaders: Headers | undefined - globalThis.fetch = (async (input, init) => { - const request = - input instanceof Request - ? input.clone() - : new Request(input as RequestInfo, init) - const body = await request.text() - if (body) requestBody = JSON.parse(body) as Record - requestHeaders = new Headers(request.headers) - return makeMessageResponse() - }) as typeof fetch - - await sideQuery({ - querySource: 'model_validation', - model: 'claude-side-attribution-test', - system, - messages: [{ role: 'user', content: 'hello' }], - }) - - if (!Array.isArray(requestBody?.system)) { - throw new Error('expected captured side-query system blocks') - } - if (!requestHeaders) { - throw new Error('expected captured side-query request headers') - } - return { - system: requestBody.system as Array>, - headers: requestHeaders, - } -} - -async function captureSideQuerySystem( - system: - | string - | Array<{ - type: 'text' - text: string - cache_control?: { type: 'ephemeral'; scope?: 'global' | 'org' } - }> = 'stable side prompt', -): Promise>> { - return (await captureSideQueryRequest(system)).system -} - -function blockTexts(blocks: Array>): string[] { - return blocks.flatMap(block => - typeof block.text === 'string' ? [block.text] : [], - ) -} - -beforeEach(async () => { - await acquireSharedMutationLock('sideQuery.attribution.test.ts') - setFlagSettingsPath(undefined) - setFlagSettingsInline(null) - setAllowedSettingSources([...SETTING_SOURCES]) - resetSettingsCache() - configRoot = mkdtempSync(join(tmpdir(), 'side-query-attribution-')) - for (const key of envKeys) delete process.env[key] - process.env.ANTHROPIC_API_KEY = 'sk-test-side-query' - process.env.OPENCLAUDE_CONFIG_DIR = configRoot - process.env.CLAUDE_FEATURE_FLAGS_FILE = join( - configRoot, - 'feature-flags.json', - ) - ;(globalThis as Record).MACRO = { - VERSION: '0.0.0-test', - DISPLAY_VERSION: '0.0.0-test', - BUILD_TIME: 'test', - ISSUES_EXPLAINER: 'test', - PACKAGE_URL: 'test', - NATIVE_PACKAGE_URL: undefined, - } - getClaudeAIOAuthTokens.cache?.clear?.() - resetGrowthBook() -}) - -function setIgnoredApiKeyHelper(): void { - delete process.env.ANTHROPIC_API_KEY - setFlagSettingsPath(undefined) - setFlagSettingsInline({ apiKeyHelper: 'ignored-test-helper' }) - setAllowedSettingSources(['flagSettings']) - resetSettingsCache() - enableConfigs() - process.env.NODE_ENV = 'development' -} - -afterEach(() => { - try { - for (const key of envKeys) { - if (originalEnv[key] === undefined) { - delete process.env[key] - } else { - process.env[key] = originalEnv[key] - } + if (!Array.isArray(requestBody?.system)) { + throw new Error('expected captured side-query system blocks') } - globalThis.fetch = originalFetch - if (originalNodeEnv === undefined) { - delete process.env.NODE_ENV - } else { - process.env.NODE_ENV = originalNodeEnv + if (!requestHeaders) { + throw new Error('expected captured side-query request headers') } + return { + system: requestBody.system as Array>, + headers: requestHeaders, + } + } + + async function captureSideQuerySystem( + system: + | string + | Array<{ + type: 'text' + text: string + cache_control?: { type: 'ephemeral'; scope?: 'global' | 'org' } + }> = 'stable side prompt', + ): Promise>> { + return (await captureSideQueryRequest(system)).system + } + + function blockTexts(blocks: Array>): string[] { + return blocks.flatMap(block => + typeof block.text === 'string' ? [block.text] : [], + ) + } + + beforeEach(async () => { + await acquireSharedMutationLock('sideQuery.attribution.test.ts') setFlagSettingsPath(undefined) setFlagSettingsInline(null) setAllowedSettingSources([...SETTING_SOURCES]) resetSettingsCache() - if (hadSavedMacro) { - ;(globalThis as Record).MACRO = savedMacro - } else { - delete (globalThis as Record).MACRO + configRoot = mkdtempSync(join(tmpdir(), 'side-query-attribution-')) + for (const key of envKeys) delete process.env[key] + process.env.ANTHROPIC_API_KEY = 'sk-test-side-query' + process.env.OPENCLAUDE_CONFIG_DIR = configRoot + process.env.CLAUDE_FEATURE_FLAGS_FILE = join( + configRoot, + 'feature-flags.json', + ) + ;(globalThis as Record).MACRO = { + VERSION: '0.0.0-test', + DISPLAY_VERSION: '0.0.0-test', + BUILD_TIME: 'test', + ISSUES_EXPLAINER: 'test', + PACKAGE_URL: 'test', + NATIVE_PACKAGE_URL: undefined, } getClaudeAIOAuthTokens.cache?.clear?.() resetGrowthBook() - if (configRoot) { - rmSync(configRoot, { recursive: true, force: true }) - configRoot = undefined - } - } finally { - releaseSharedMutationLock() + }) + + function setIgnoredApiKeyHelper(): void { + delete process.env.ANTHROPIC_API_KEY + setFlagSettingsPath(undefined) + setFlagSettingsInline({ apiKeyHelper: 'ignored-test-helper' }) + setAllowedSettingSources(['flagSettings']) + resetSettingsCache() + enableConfigs() + process.env.NODE_ENV = 'development' } -}) -if (runInProviderIsolatedChild) { - test('runs side-query attribution cases with the real provider module', async () => { - await runTestFileWithRealProviders(import.meta.path) - }, { timeout: REAL_PROVIDER_TEST_TIMEOUT_MS + 5_000 }) -} - -const describeAttribution = runInProviderIsolatedChild - ? describe.skip - : describe - -describeAttribution('sideQuery Anthropic attribution', () => { - for (const [label, envKey, envValue, ambientAuth] of [ - ['remote', 'CLAUDE_CODE_REMOTE', '1', 'api-key'], - [ - 'Claude Desktop', - 'CLAUDE_CODE_ENTRYPOINT', - 'claude-desktop', - 'api-key-helper', - ], - [ - 'Unix-socket OAuth proxy', - 'ANTHROPIC_UNIX_SOCKET', - '/tmp/openclaude-auth-test.sock', - 'auth-token', - ], - ] as const) { - test(`uses OAuth headers and attribution in managed ${label} sessions`, async () => { - process.env[envKey] = envValue - process.env.CLAUDE_CODE_OAUTH_TOKEN = 'oauth-test-token' - process.env.CLAUDE_CODE_ATTRIBUTION_HEADER = '0' - if (ambientAuth === 'api-key-helper') setIgnoredApiKeyHelper() - if (ambientAuth === 'auth-token') { - process.env.ANTHROPIC_AUTH_TOKEN = 'ignored-test-auth-token' + afterEach(() => { + try { + for (const key of envKeys) { + if (originalEnv[key] === undefined) { + delete process.env[key] + } else { + process.env[key] = originalEnv[key] + } } + globalThis.fetch = originalFetch + if (originalNodeEnv === undefined) { + delete process.env.NODE_ENV + } else { + process.env.NODE_ENV = originalNodeEnv + } + setFlagSettingsPath(undefined) + setFlagSettingsInline(null) + setAllowedSettingSources([...SETTING_SOURCES]) + resetSettingsCache() + if (hadSavedMacro) { + ;(globalThis as Record).MACRO = savedMacro + } else { + delete (globalThis as Record).MACRO + } + getClaudeAIOAuthTokens.cache?.clear?.() + resetGrowthBook() + if (configRoot) { + rmSync(configRoot, { recursive: true, force: true }) + configRoot = undefined + } + } finally { + releaseSharedMutationLock() + } + }) + + describe('sideQuery Anthropic attribution', () => { + for (const [label, envKey, envValue, ambientAuth] of [ + ['remote', 'CLAUDE_CODE_REMOTE', '1', 'api-key'], + [ + 'Claude Desktop', + 'CLAUDE_CODE_ENTRYPOINT', + 'claude-desktop', + 'api-key-helper', + ], + [ + 'Unix-socket OAuth proxy', + 'ANTHROPIC_UNIX_SOCKET', + '/tmp/openclaude-auth-test.sock', + 'auth-token', + ], + ] as const) { + test(`uses OAuth headers and attribution in managed ${label} sessions`, async () => { + process.env[envKey] = envValue + process.env.CLAUDE_CODE_OAUTH_TOKEN = 'oauth-test-token' + process.env.CLAUDE_CODE_ATTRIBUTION_HEADER = '0' + if (ambientAuth === 'api-key-helper') setIgnoredApiKeyHelper() + if (ambientAuth === 'auth-token') { + process.env.ANTHROPIC_AUTH_TOKEN = 'ignored-test-auth-token' + } + getClaudeAIOAuthTokens.cache?.clear?.() + + const request = await captureSideQueryRequest() + const texts = blockTexts(request.system) + + expect( + texts.some(text => text.startsWith('x-anthropic-billing-header')), + ).toBe(true) + expect(request.headers.get('authorization')).toBe( + 'Bearer oauth-test-token', + ) + expect(request.headers.has('x-api-key')).toBe(false) + }) + } + + test('keeps Unix-socket API-key auth when the OAuth placeholder is absent', async () => { + writeFileSync( + join(configRoot!, '.credentials.json'), + JSON.stringify({ + claudeAiOauth: { + accessToken: 'stored-test-oauth-token', + refreshToken: null, + expiresAt: null, + scopes: ['user:inference'], + subscriptionType: null, + rateLimitTier: null, + }, + }), + ) + process.env.ANTHROPIC_UNIX_SOCKET = '/tmp/openclaude-auth-test.sock' + process.env.CLAUDE_CODE_ATTRIBUTION_HEADER = '0' getClaudeAIOAuthTokens.cache?.clear?.() const request = await captureSideQueryRequest() @@ -272,101 +283,71 @@ describeAttribution('sideQuery Anthropic attribution', () => { expect( texts.some(text => text.startsWith('x-anthropic-billing-header')), - ).toBe(true) - expect(request.headers.get('authorization')).toBe( - 'Bearer oauth-test-token', + ).toBe(false) + expect(request.headers.get('x-api-key')).toBe('sk-test-side-query') + expect(request.headers.has('authorization')).toBe(false) + }) + + test('strips the block from a custom native endpoint', async () => { + process.env.ANTHROPIC_BASE_URL = 'https://custom-anthropic.example/v1' + + const texts = blockTexts( + await captureSideQuerySystem([ + { type: 'text', text: 'x-anthropic-billing-header: stale' }, + { type: 'text', text: 'stable side prompt' }, + ]), ) - expect(request.headers.has('x-api-key')).toBe(false) + + expect( + texts.some(text => text.startsWith('x-anthropic-billing-header')), + ).toBe(false) + expect(texts).toContain('stable side prompt') }) - } - test('keeps Unix-socket API-key auth when the OAuth placeholder is absent', async () => { - writeFileSync( - join(configRoot!, '.credentials.json'), - JSON.stringify({ - claudeAiOauth: { - accessToken: 'stored-test-oauth-token', - refreshToken: null, - expiresAt: null, - scopes: ['user:inference'], - subscriptionType: null, - rateLimitTier: null, - }, - }), - ) - process.env.ANTHROPIC_UNIX_SOCKET = '/tmp/openclaude-auth-test.sock' - process.env.CLAUDE_CODE_ATTRIBUTION_HEADER = '0' - getClaudeAIOAuthTokens.cache?.clear?.() + test('honors the disabled setting for an official API key', async () => { + process.env.CLAUDE_CODE_ATTRIBUTION_HEADER = '0' - const request = await captureSideQueryRequest() - const texts = blockTexts(request.system) + const texts = blockTexts(await captureSideQuerySystem()) - expect( - texts.some(text => text.startsWith('x-anthropic-billing-header')), - ).toBe(false) - expect(request.headers.get('x-api-key')).toBe('sk-test-side-query') - expect(request.headers.has('authorization')).toBe(false) - }) + expect( + texts.some(text => text.startsWith('x-anthropic-billing-header')), + ).toBe(false) + expect(texts).toContain('stable side prompt') + }) - test('strips the block from a custom native endpoint', async () => { - process.env.ANTHROPIC_BASE_URL = 'https://custom-anthropic.example/v1' + test('keeps the block for official OAuth when globally disabled', async () => { + delete process.env.ANTHROPIC_API_KEY + process.env.CLAUDE_CODE_OAUTH_TOKEN = 'oauth-test-token' + process.env.CLAUDE_CODE_ATTRIBUTION_HEADER = '0' + getClaudeAIOAuthTokens.cache?.clear?.() - const texts = blockTexts( - await captureSideQuerySystem([ + const texts = blockTexts(await captureSideQuerySystem()) + + expect( + texts.some(text => text.startsWith('x-anthropic-billing-header')), + ).toBe(true) + expect(texts).toContain('stable side prompt') + }) + + test('keeps one generated block and preserves cache metadata order', async () => { + const blocks = await captureSideQuerySystem([ { type: 'text', text: 'x-anthropic-billing-header: stale' }, - { type: 'text', text: 'stable side prompt' }, - ]), - ) + { + type: 'text', + text: 'stable cached side prompt', + cache_control: { type: 'ephemeral', scope: 'org' }, + }, + ]) - expect( - texts.some(text => text.startsWith('x-anthropic-billing-header')), - ).toBe(false) - expect(texts).toContain('stable side prompt') - }) - - test('honors the disabled setting for an official API key', async () => { - process.env.CLAUDE_CODE_ATTRIBUTION_HEADER = '0' - - const texts = blockTexts(await captureSideQuerySystem()) - - expect( - texts.some(text => text.startsWith('x-anthropic-billing-header')), - ).toBe(false) - expect(texts).toContain('stable side prompt') - }) - - test('keeps the block for official OAuth when globally disabled', async () => { - delete process.env.ANTHROPIC_API_KEY - process.env.CLAUDE_CODE_OAUTH_TOKEN = 'oauth-test-token' - process.env.CLAUDE_CODE_ATTRIBUTION_HEADER = '0' - getClaudeAIOAuthTokens.cache?.clear?.() - - const texts = blockTexts(await captureSideQuerySystem()) - - expect( - texts.some(text => text.startsWith('x-anthropic-billing-header')), - ).toBe(true) - expect(texts).toContain('stable side prompt') - }) - - test('keeps one generated block and preserves cache metadata order', async () => { - const blocks = await captureSideQuerySystem([ - { type: 'text', text: 'x-anthropic-billing-header: stale' }, - { - type: 'text', - text: 'stable cached side prompt', - cache_control: { type: 'ephemeral', scope: 'org' }, - }, - ]) - - const texts = blockTexts(blocks) - expect( - texts.filter(text => text.startsWith('x-anthropic-billing-header')), - ).toHaveLength(1) - expect(texts[1]).toBe('stable cached side prompt') - expect(blocks[1]?.cache_control).toEqual({ - type: 'ephemeral', - scope: 'org', + const texts = blockTexts(blocks) + expect( + texts.filter(text => text.startsWith('x-anthropic-billing-header')), + ).toHaveLength(1) + expect(texts[1]).toBe('stable cached side prompt') + expect(blocks[1]?.cache_control).toEqual({ + type: 'ephemeral', + scope: 'org', + }) }) }) -}) +}