From 4cf981200fd881d839a7872cfb4f342232eeaa37 Mon Sep 17 00:00:00 2001 From: Bogdan Date: Thu, 18 Jun 2026 02:42:41 +0200 Subject: [PATCH] feat(cache): classify prompt-cache breaks by reliability (#1693) * feat(cache): classify prompt-cache breaks by reliability * fix(cache): stabilize prompt-cache break metadata detection * fix(cache): honor legacy OpenAI base fallback * fix(cache): normalize OpenAI base URL hints * fix(cache): align cache-break provider flag truthiness * fix(cache): ignore undefined OpenAI base hints * fix(cache): sanitize prompt cache route labels --- src/services/api/cacheMetrics.test.ts | 28 + src/services/api/cacheMetrics.ts | 17 + .../api/promptCacheBreakDetection.test.ts | 575 ++++++++++++++++++ src/services/api/promptCacheBreakDetection.ts | 316 +++++++++- 4 files changed, 928 insertions(+), 8 deletions(-) create mode 100644 src/services/api/promptCacheBreakDetection.test.ts diff --git a/src/services/api/cacheMetrics.test.ts b/src/services/api/cacheMetrics.test.ts index 893924c8e..c98d2f394 100644 --- a/src/services/api/cacheMetrics.test.ts +++ b/src/services/api/cacheMetrics.test.ts @@ -6,6 +6,7 @@ import { formatCacheMetricsCompact, formatCacheMetricsFull, addCacheMetrics, + getCacheMetricsReliability, } from './cacheMetrics.js' describe('extractCacheMetrics — Anthropic (firstParty/bedrock/vertex/foundry)', () => { @@ -447,6 +448,33 @@ describe('resolveCacheProvider — .localhost TLD (RFC 6761)', () => { }) }) +describe('getCacheMetricsReliability', () => { + test('Anthropic-native cache metrics are reliable', () => { + expect(getCacheMetricsReliability('anthropic')).toBe('reliable') + expect(getCacheMetricsReliability('copilot-claude')).toBe('reliable') + }) + + test('OpenAI-compatible cache metrics are advisory by default', () => { + expect( + getCacheMetricsReliability( + resolveCacheProvider('openai', { + openAiBaseUrl: 'https://api.openai.com/v1', + }), + ), + ).toBe('advisory') + expect(getCacheMetricsReliability('codex')).toBe('advisory') + expect(getCacheMetricsReliability('kimi')).toBe('advisory') + expect(getCacheMetricsReliability('deepseek')).toBe('advisory') + expect(getCacheMetricsReliability('gemini')).toBe('advisory') + expect(getCacheMetricsReliability('self-hosted')).toBe('advisory') + }) + + test('providers without cache metric support remain unsupported', () => { + expect(getCacheMetricsReliability('copilot')).toBe('unsupported') + expect(getCacheMetricsReliability('ollama')).toBe('unsupported') + }) +}) + describe('extractCacheMetrics — hit rate clamp', () => { test('hitRate is clamped to 1.0 on pathological input (read > total)', () => { // Defensive guard: with valid non-negative inputs the math enforces diff --git a/src/services/api/cacheMetrics.ts b/src/services/api/cacheMetrics.ts index d1f277948..27acf1e88 100644 --- a/src/services/api/cacheMetrics.ts +++ b/src/services/api/cacheMetrics.ts @@ -65,6 +65,11 @@ export type CacheAwareProvider = | 'copilot' | 'copilot-claude' +export type CacheMetricsReliability = + | 'reliable' + | 'advisory' + | 'unsupported' + /** Unified cache metrics for one API response. */ export type CacheMetrics = { /** Tokens served from cache on this request. */ @@ -273,6 +278,18 @@ export function resolveCacheProvider( return 'openai' } +export function getCacheMetricsReliability( + provider: CacheAwareProvider, +): CacheMetricsReliability { + if (provider === 'copilot' || provider === 'ollama') { + return 'unsupported' + } + if (provider === 'anthropic' || provider === 'copilot-claude') { + return 'reliable' + } + return 'advisory' +} + /** * Read the cached-tokens count from a RAW provider usage object, handling * every shape we know about. Callers are the shim layer (openaiShim, diff --git a/src/services/api/promptCacheBreakDetection.test.ts b/src/services/api/promptCacheBreakDetection.test.ts new file mode 100644 index 000000000..5ad9423e3 --- /dev/null +++ b/src/services/api/promptCacheBreakDetection.test.ts @@ -0,0 +1,575 @@ +import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test' +import type { BetaToolUnion } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs' +import type { TextBlockParam } from '@anthropic-ai/sdk/resources/index.mjs' +import type { QuerySource } from '../../constants/querySource.js' +import type { Message } from '../../types/message.js' +import type { DebugLogLevel } from '../../utils/debug.js' +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../../test/sharedMutationLock.js' + +type PromptCacheBreakModule = typeof import('./promptCacheBreakDetection.js') +type EventCall = { + name: string + metadata: Record +} +type DebugCall = { + message: string + options?: { level?: DebugLogLevel } +} + +const events: EventCall[] = [] +const debugCalls: DebugCall[] = [] + +const logEventMock = mock((name: string, metadata: Record) => { + events.push({ name, metadata }) +}) +const logForDebuggingMock = mock( + (message: string, options?: { level?: DebugLogLevel }) => { + debugCalls.push({ message, options }) + }, +) +const actualDebugModule = await import('../../utils/debug.js') + +mock.module('../analytics/index.js', () => ({ + logEvent: logEventMock, +})) + +mock.module('src/utils/debug.js', () => ({ + ...actualDebugModule, + logForDebugging: logForDebuggingMock, +})) + +mock.module('../../utils/debug.js', () => ({ + ...actualDebugModule, + logForDebugging: logForDebuggingMock, +})) + +const PROVIDER_ENV_KEYS = [ + 'CLAUDE_CODE_USE_OPENAI', + 'CLAUDE_CODE_USE_GITHUB', + 'CLAUDE_CODE_USE_GEMINI', + 'CLAUDE_CODE_USE_MISTRAL', + 'CLAUDE_CODE_USE_BEDROCK', + 'CLAUDE_CODE_USE_VERTEX', + 'CLAUDE_CODE_USE_FOUNDRY', + 'OPENAI_BASE_URL', + 'OPENAI_API_BASE', + 'OPENAI_MODEL', +] as const + +const originalEnv: Record = {} +let detector: PromptCacheBreakModule | undefined + +for (const key of PROVIDER_ENV_KEYS) { + originalEnv[key] = process.env[key] +} + +async function loadDetector(): Promise { + detector ??= await import('./promptCacheBreakDetection.js') + return detector +} + +function restoreProviderEnv(): void { + for (const key of PROVIDER_ENV_KEYS) { + const value = originalEnv[key] + if (value === undefined) { + delete process.env[key] + } else { + process.env[key] = value + } + } +} + +function clearProviderEnv(): void { + for (const key of PROVIDER_ENV_KEYS) { + delete process.env[key] + } +} + +function useOpenAIProvider(): void { + clearProviderEnv() + process.env.CLAUDE_CODE_USE_OPENAI = '1' + process.env.OPENAI_BASE_URL = 'https://api.openai.com/v1' + process.env.OPENAI_MODEL = 'gpt-5.5' +} + +function useOpenAIProviderWithDisabledFoundryFlag(flagValue: string): void { + useOpenAIProvider() + process.env.CLAUDE_CODE_USE_FOUNDRY = flagValue +} + +function useOpenAIProviderWithWhitespaceBaseFallback(): void { + clearProviderEnv() + process.env.CLAUDE_CODE_USE_OPENAI = '1' + process.env.OPENAI_BASE_URL = ' ' + process.env.OPENAI_API_BASE = 'https://api.deepseek.com/v1' + process.env.OPENAI_MODEL = 'deepseek-chat' +} + +function useCodexAliasWithLiteralUndefinedBaseUrl(): void { + clearProviderEnv() + process.env.CLAUDE_CODE_USE_OPENAI = '1' + process.env.OPENAI_BASE_URL = 'undefined' + process.env.OPENAI_MODEL = 'codexplan' +} + +function useOpenRouterProvider(): void { + clearProviderEnv() + process.env.CLAUDE_CODE_USE_OPENAI = '1' + process.env.OPENAI_BASE_URL = 'https://openrouter.ai/api/v1' + process.env.OPENAI_MODEL = 'gpt-4o' +} + +function useUnsupportedGithubProvider(): void { + clearProviderEnv() + process.env.CLAUDE_CODE_USE_GITHUB = '1' + process.env.OPENAI_MODEL = 'gpt-4o' +} + +function useFoundryProvider(): void { + clearProviderEnv() + process.env.CLAUDE_CODE_USE_FOUNDRY = '1' +} + +function systemBlock( + text: string, + cacheControl?: Record, +): TextBlockParam[] { + return [ + { + type: 'text', + text, + ...(cacheControl ? { cache_control: cacheControl } : {}), + }, + ] as TextBlockParam[] +} + +function tool( + name: string, + schema: Record = { + type: 'object', + properties: {}, + }, +): BetaToolUnion { + return { + type: 'custom', + name, + description: `${name} test tool`, + input_schema: schema, + } as unknown as BetaToolUnion +} + +function assistantMessages(gapMs: number): Message[] { + return [ + { + type: 'assistant', + timestamp: new Date(Date.now() - gapMs).toISOString(), + uuid: '00000000-0000-4000-8000-000000000001', + message: { + role: 'assistant', + content: [], + id: 'msg_1', + model: 'claude-sonnet-4', + usage: { + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + }, + }, + ] as unknown as Message[] +} + +function snapshot( + overrides: Partial[0]> = {}, +): Parameters[0] { + return { + system: systemBlock('base system prompt'), + toolSchemas: [tool('Read')], + querySource: 'repl_main_thread' as QuerySource, + model: 'claude-sonnet-4', + ...overrides, + } +} + +async function triggerCacheDrop({ + first = snapshot(), + second = snapshot(), + secondMessages = assistantMessages(60_000), +}: { + first?: Parameters[0] + second?: Parameters[0] + secondMessages?: Message[] +} = {}): Promise { + const mod = await loadDetector() + mod.recordPromptState(first) + await mod.checkResponseForCacheBreak( + first.querySource, + 10_000, + 0, + assistantMessages(60_000), + first.agentId, + 'req-prev', + ) + + mod.recordPromptState(second) + await mod.checkResponseForCacheBreak( + second.querySource, + 1_000, + 0, + secondMessages, + second.agentId, + 'req-break', + ) + + const event = events.findLast(e => e.name === 'tengu_prompt_cache_break') + expect(event).toBeDefined() + return event! +} + +beforeEach(async () => { + await acquireSharedMutationLock('promptCacheBreakDetection.test.ts') + events.length = 0 + debugCalls.length = 0 + logEventMock.mockClear() + logForDebuggingMock.mockClear() + clearProviderEnv() + const mod = await loadDetector() + mod.resetPromptCacheBreakDetection() +}) + +afterEach(() => { + try { + detector?.resetPromptCacheBreakDetection() + restoreProviderEnv() + } finally { + releaseSharedMutationLock() + } +}) + +describe('prompt cache break taxonomy', () => { + test('tool addition/removal classifies as expected tool schema change', async () => { + const event = await triggerCacheDrop({ + first: snapshot({ toolSchemas: [tool('Read'), tool('Edit')] }), + second: snapshot({ toolSchemas: [tool('Edit'), tool('Bash')] }), + }) + + expect(event.metadata).toMatchObject({ + classification: 'expected_tool_schema_change', + addedToolCount: 1, + removedToolCount: 1, + addedTools: 'Bash', + removedTools: 'Read', + }) + }) + + test('changed MCP tool schema names are sanitized in analytics', async () => { + const event = await triggerCacheDrop({ + first: snapshot({ + toolSchemas: [ + tool('mcp__repo_server__search', { + type: 'object', + properties: { query: { type: 'string' } }, + }), + ], + }), + second: snapshot({ + toolSchemas: [ + tool('mcp__repo_server__search', { + type: 'object', + properties: { query: { type: 'string' }, limit: { type: 'number' } }, + }), + ], + }), + }) + + expect(event.metadata).toMatchObject({ + classification: 'expected_tool_schema_change', + changedToolSchemas: 'mcp', + }) + }) + + test('system prompt hash change classifies as expected local prompt change', async () => { + const event = await triggerCacheDrop({ + first: snapshot({ system: systemBlock('base system prompt') }), + second: snapshot({ system: systemBlock('updated system prompt') }), + }) + + expect(event.metadata).toMatchObject({ + classification: 'expected_local_prompt_change', + severity: 'info', + }) + }) + + test('cache-control-only change classifies separately', async () => { + const event = await triggerCacheDrop({ + first: snapshot({ + system: systemBlock('base system prompt', { type: 'ephemeral' }), + }), + second: snapshot({ + system: systemBlock('base system prompt', { + type: 'ephemeral', + ttl: '1h', + }), + }), + }) + + expect(event.metadata).toMatchObject({ + classification: 'expected_cache_control_change', + cacheControlChanged: true, + systemPromptChanged: false, + }) + }) + + test('global cache strategy change classifies with cache-control changes', async () => { + const event = await triggerCacheDrop({ + first: snapshot({ globalCacheStrategy: 'tool_based' }), + second: snapshot({ globalCacheStrategy: 'system_prompt' }), + }) + + expect(event.metadata).toMatchObject({ + classification: 'expected_cache_control_change', + cacheControlChanged: false, + globalCacheStrategyChanged: true, + prevGlobalCacheStrategy: 'tool_based', + newGlobalCacheStrategy: 'system_prompt', + }) + }) + + test('model, beta, effort, and extra-body changes classify as expected model or beta change', async () => { + const event = await triggerCacheDrop({ + first: snapshot({ + model: 'claude-sonnet-4', + betas: ['beta-a'], + effortValue: 'medium', + extraBodyParams: { metadata: { source: 'first' } }, + }), + second: snapshot({ + model: 'claude-opus-4', + betas: ['beta-b'], + effortValue: 'high', + extraBodyParams: { metadata: { source: 'second' } }, + }), + }) + + expect(event.metadata).toMatchObject({ + classification: 'expected_model_or_beta_change', + modelChanged: true, + betasChanged: true, + effortChanged: true, + extraBodyChanged: true, + addedBetas: 'beta-b', + removedBetas: 'beta-a', + }) + }) + + test('same prompt/schema under TTL on advisory provider classifies as provider cache instability with info severity', async () => { + useOpenAIProvider() + + const event = await triggerCacheDrop() + const debug = debugCalls.findLast(call => + call.message.startsWith('[PROMPT CACHE BREAK]'), + ) + + expect(event.metadata).toMatchObject({ + classification: 'provider_cache_instability', + cacheMetricsReliability: 'advisory', + severity: 'info', + prevCacheReadTokens: 10_000, + cacheReadTokens: 1_000, + tokenDrop: 9_000, + querySource: 'repl_main_thread', + model: 'claude-sonnet-4', + providerRoute: 'openai', + requestId: 'req-break', + }) + expect(debug?.options).toEqual({ level: 'info' }) + expect(debug?.message).not.toMatch(/app crash|crash|local mutation/i) + }) + + for (const falseyFoundryFlag of ['off', 'no'] as const) { + test(`stale Foundry=${falseyFoundryFlag} env flag does not override OpenAI cache metadata`, async () => { + useOpenAIProviderWithDisabledFoundryFlag(falseyFoundryFlag) + + const event = await triggerCacheDrop() + const debug = debugCalls.findLast(call => + call.message.startsWith('[PROMPT CACHE BREAK]'), + ) + + expect(event.metadata).toMatchObject({ + classification: 'provider_cache_instability', + cacheMetricsReliability: 'advisory', + cacheProvider: 'openai', + providerRoute: 'openai', + severity: 'info', + }) + expect(debug?.options).toEqual({ level: 'info' }) + }) + } + + test('blank OpenAI base URL falls back to legacy API base for cache provider metadata', async () => { + useOpenAIProviderWithWhitespaceBaseFallback() + + const event = await triggerCacheDrop({ + first: snapshot({ model: 'deepseek-chat' }), + second: snapshot({ model: 'deepseek-chat' }), + }) + + expect(event.metadata).toMatchObject({ + classification: 'provider_cache_instability', + cacheMetricsReliability: 'advisory', + cacheProvider: 'deepseek', + severity: 'info', + }) + }) + + test('literal undefined OpenAI base URL still reports Codex alias metadata', async () => { + useCodexAliasWithLiteralUndefinedBaseUrl() + + const event = await triggerCacheDrop({ + first: snapshot({ model: 'codexplan' }), + second: snapshot({ model: 'codexplan' }), + }) + + expect(event.metadata).toMatchObject({ + classification: 'provider_cache_instability', + cacheMetricsReliability: 'advisory', + cacheProvider: 'codex', + providerRoute: 'codex', + severity: 'info', + }) + }) + + test('descriptor OpenAI-compatible route IDs are normalized before logging', async () => { + useOpenRouterProvider() + + const event = await triggerCacheDrop({ + first: snapshot({ model: 'gpt-4o' }), + second: snapshot({ model: 'gpt-4o' }), + }) + const debug = debugCalls.findLast(call => + call.message.startsWith('[PROMPT CACHE BREAK]'), + ) + + expect(event.metadata).toMatchObject({ + classification: 'provider_cache_instability', + cacheMetricsReliability: 'advisory', + cacheProvider: 'openai', + providerRoute: 'openai-compatible', + severity: 'info', + }) + expect(debug?.message).toContain('route=openai-compatible') + expect(debug?.message).not.toContain('openrouter') + }) + + test('TTL-window cache drops classify as possible TTL expiry', async () => { + const event = await triggerCacheDrop({ + secondMessages: assistantMessages(61 * 60 * 1000), + }) + + expect(event.metadata).toMatchObject({ + classification: 'possible_ttl_expiry', + lastAssistantMsgOver5minAgo: true, + lastAssistantMsgOver1hAgo: true, + }) + }) + + test('unknown or incomplete state classifies as unknown local mutation, not provider-side instability', async () => { + const event = await triggerCacheDrop({ + secondMessages: [], + }) + + expect(event.metadata).toMatchObject({ + classification: 'unknown_local_mutation', + timeSinceLastAssistantMsg: -1, + }) + }) + + test('unsupported provider metrics classify as metrics unavailable', async () => { + useUnsupportedGithubProvider() + + const event = await triggerCacheDrop({ + first: snapshot({ model: 'gpt-4o' }), + second: snapshot({ model: 'gpt-4o' }), + }) + + expect(event.metadata).toMatchObject({ + classification: 'metrics_unavailable', + cacheMetricsReliability: 'unsupported', + providerRoute: 'github', + }) + }) + + test('unsupported provider metrics do not hide known local prompt changes', async () => { + useUnsupportedGithubProvider() + + const event = await triggerCacheDrop({ + first: snapshot({ + model: 'gpt-4o', + system: systemBlock('base system prompt'), + }), + second: snapshot({ + model: 'gpt-4o', + system: systemBlock('changed system prompt'), + }), + }) + + expect(event.metadata).toMatchObject({ + classification: 'expected_local_prompt_change', + cacheMetricsReliability: 'unsupported', + systemPromptChanged: true, + }) + }) + + test('legacy Foundry provider does not get mislabeled as anthropic route', async () => { + useFoundryProvider() + + const event = await triggerCacheDrop() + const debug = debugCalls.findLast(call => + call.message.startsWith('[PROMPT CACHE BREAK]'), + ) + + expect(event.metadata).toMatchObject({ + classification: 'provider_cache_instability', + cacheMetricsReliability: 'reliable', + cacheProvider: 'anthropic', + providerRoute: 'foundry', + severity: 'warning', + }) + expect(debug?.options).toEqual({ level: 'warn' }) + }) + + test('cache deletion expected drops remain suppressed', async () => { + const mod = await loadDetector() + const first = snapshot() + + mod.recordPromptState(first) + await mod.checkResponseForCacheBreak( + first.querySource, + 10_000, + 0, + assistantMessages(60_000), + first.agentId, + 'req-prev', + ) + + mod.recordPromptState(snapshot()) + mod.notifyCacheDeletion(first.querySource, first.agentId) + await mod.checkResponseForCacheBreak( + first.querySource, + 1_000, + 0, + assistantMessages(60_000), + first.agentId, + 'req-drop', + ) + + expect(events.filter(e => e.name === 'tengu_prompt_cache_break')).toEqual([]) + expect( + debugCalls.some(call => call.message.includes('cache deletion applied')), + ).toBe(true) + }) +}) diff --git a/src/services/api/promptCacheBreakDetection.ts b/src/services/api/promptCacheBreakDetection.ts index 1599d537e..43e099ea4 100644 --- a/src/services/api/promptCacheBreakDetection.ts +++ b/src/services/api/promptCacheBreakDetection.ts @@ -5,16 +5,25 @@ import { mkdir, writeFile } from 'fs/promises' import { join } from 'path' import type { AgentId } from 'src/types/ids.js' import type { Message } from 'src/types/message.js' -import { logForDebugging } from 'src/utils/debug.js' +import { type DebugLogLevel, logForDebugging } from 'src/utils/debug.js' import { djb2Hash } from 'src/utils/hash.js' import { logError } from 'src/utils/log.js' +import type { APIProvider } from 'src/utils/model/providers.js' import { getClaudeTempDir } from 'src/utils/permissions/filesystem.js' -import { jsonStringify } from 'src/utils/slowOperations.js' import type { QuerySource } from '../../constants/querySource.js' +import { + getTransportKindForRoute, + resolveActiveRouteIdFromEnv, +} from '../../integrations/routeMetadata.js' import { type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, logEvent, } from '../analytics/index.js' +import { + getCacheMetricsReliability, + resolveCacheProvider, + type CacheMetricsReliability, +} from './cacheMetrics.js' function getCacheBreakDiffPath(): string { const chars = 'abcdefghijklmnopqrstuvwxyz0123456789' @@ -98,6 +107,24 @@ type PendingChanges = { buildPrevDiffableContent: () => string } +export type PromptCacheBreakKind = + | 'expected_local_prompt_change' + | 'expected_tool_schema_change' + | 'expected_cache_control_change' + | 'expected_model_or_beta_change' + | 'provider_cache_instability' + | 'possible_ttl_expiry' + | 'unknown_local_mutation' + | 'metrics_unavailable' + +type PromptCacheBreakSeverity = 'debug' | 'info' | 'warning' + +type PromptCacheBreakClassification = { + kind: PromptCacheBreakKind + severity: PromptCacheBreakSeverity + debugLevel: DebugLogLevel +} + const previousStateBySource = new Map() // Cap the number of tracked sources to prevent unbounded memory growth. @@ -168,7 +195,7 @@ function stripCacheControl( } function computeHash(data: unknown): number { - const str = jsonStringify(data) + const str = stringifyCacheBreakData(data) if (typeof Bun !== 'undefined') { const hash = Bun.hash(str) // Bun.hash can return bigint for large inputs; convert to number safely @@ -178,6 +205,40 @@ function computeHash(data: unknown): number { return djb2Hash(str) } +function stringifyCacheBreakData(data: unknown): string { + return JSON.stringify(data) ?? '' +} + +function isCacheBreakEnvTruthy(value: string | undefined): boolean { + const normalized = value?.trim().toLowerCase() + return ( + normalized === '1' || + normalized === 'true' || + normalized === 'yes' || + normalized === 'on' + ) +} + +function getNonEmptyEnvValue(value: string | undefined): string | undefined { + const trimmed = value?.trim() + return trimmed && trimmed !== 'undefined' ? trimmed : undefined +} + +const SAFE_PROMPT_CACHE_PROVIDER_ROUTES = new Set([ + 'anthropic', + 'openai', + 'custom', + 'gemini', + 'mistral', + 'github', + 'bedrock', + 'vertex', + 'nvidia-nim', + 'minimax', + 'xiaomi-mimo', + 'xai', +]) + /** MCP tool names are user-controlled (server config) and may leak filepaths. * Collapse them to 'mcp'; built-in names are a fixed vocabulary. */ function sanitizeToolName(name: string): string { @@ -213,7 +274,8 @@ function buildDiffableContent( .map(t => { if (!('name' in t)) return 'unknown' const desc = 'description' in t ? t.description : '' - const schema = 'input_schema' in t ? jsonStringify(t.input_schema) : '' + const schema = + 'input_schema' in t ? stringifyCacheBreakData(t.input_schema) : '' return `${t.name}\n description: ${desc}\n input_schema: ${schema}` }) .sort() @@ -221,6 +283,211 @@ function buildDiffableContent( return `Model: ${model}\n\n=== System Prompt ===\n\n${systemText}\n\n=== Tools (${tools.length}) ===\n\n${toolDetails}\n` } +function classifyPromptCacheBreak({ + changes, + cacheMetricsReliability, + lastAssistantMsgOver5minAgo, + timeSinceLastAssistantMsg, +}: { + changes: PendingChanges | null + cacheMetricsReliability: CacheMetricsReliability + lastAssistantMsgOver5minAgo: boolean + timeSinceLastAssistantMsg: number | null +}): PromptCacheBreakClassification { + if (changes) { + if (changes.systemPromptChanged) { + return { + kind: 'expected_local_prompt_change', + severity: 'info', + debugLevel: 'info', + } + } + if (changes.toolSchemasChanged) { + return { + kind: 'expected_tool_schema_change', + severity: 'info', + debugLevel: 'info', + } + } + if ( + (changes.cacheControlChanged || changes.globalCacheStrategyChanged) && + !changes.systemPromptChanged + ) { + return { + kind: 'expected_cache_control_change', + severity: 'info', + debugLevel: 'info', + } + } + if ( + changes.modelChanged || + changes.fastModeChanged || + changes.betasChanged || + changes.autoModeChanged || + changes.overageChanged || + changes.cachedMCChanged || + changes.effortChanged || + changes.extraBodyChanged + ) { + return { + kind: 'expected_model_or_beta_change', + severity: 'info', + debugLevel: 'info', + } + } + } + + if (cacheMetricsReliability === 'unsupported') { + return { + kind: 'metrics_unavailable', + severity: 'info', + debugLevel: 'info', + } + } + + if (lastAssistantMsgOver5minAgo) { + return { + kind: 'possible_ttl_expiry', + severity: 'info', + debugLevel: 'info', + } + } + + if (timeSinceLastAssistantMsg !== null) { + const advisory = cacheMetricsReliability === 'advisory' + return { + kind: 'provider_cache_instability', + severity: advisory ? 'info' : 'warning', + debugLevel: advisory ? 'info' : 'warn', + } + } + + return { + kind: 'unknown_local_mutation', + severity: 'warning', + debugLevel: 'warn', + } +} + +function getPromptCacheBreakProviderMetadata(model: string): { + cacheProvider: string + cacheMetricsReliability: CacheMetricsReliability + providerRoute: string +} { + const activeRouteId = resolveActiveRouteIdFromEnv(process.env) + const apiProvider = resolvePromptCacheBreakAPIProvider( + process.env, + activeRouteId, + model, + ) + const cacheProvider = resolveCacheProvider(apiProvider, { + githubNativeAnthropic: isGithubNativeAnthropicModeForCacheBreak( + process.env, + model, + ), + openAiBaseUrl: + getNonEmptyEnvValue(process.env.OPENAI_BASE_URL) ?? + getNonEmptyEnvValue(process.env.OPENAI_API_BASE), + }) + return { + cacheProvider, + cacheMetricsReliability: getCacheMetricsReliability(cacheProvider), + providerRoute: getPromptCacheBreakProviderRoute( + activeRouteId, + apiProvider, + ), + } +} + +function getPromptCacheBreakProviderRoute( + activeRouteId: string | null, + apiProvider: APIProvider, +): string { + if (apiProvider === 'codex') { + return 'codex' + } + if (activeRouteId === 'anthropic' && apiProvider !== 'firstParty') { + return apiProvider + } + if (!activeRouteId) { + return apiProvider + } + if (SAFE_PROMPT_CACHE_PROVIDER_ROUTES.has(activeRouteId)) { + return activeRouteId + } + return getTransportKindForRoute(activeRouteId) ?? apiProvider +} + +function resolvePromptCacheBreakAPIProvider( + env: NodeJS.ProcessEnv, + activeRouteId: string | null, + model: string, +): APIProvider { + if (isCacheBreakEnvTruthy(env.CLAUDE_CODE_USE_FOUNDRY)) { + return 'foundry' + } + + switch (activeRouteId) { + case 'gemini': + case 'mistral': + case 'github': + case 'bedrock': + case 'vertex': + case 'nvidia-nim': + case 'minimax': + case 'xiaomi-mimo': + case 'xai': + return activeRouteId + case 'openai': + case 'custom': + if (isCacheBreakEnvTruthy(env.NVIDIA_NIM)) { + return 'nvidia-nim' + } + return isCodexCacheBreakRoute(env, model) ? 'codex' : 'openai' + case 'anthropic': + case null: + if (isCacheBreakEnvTruthy(env.NVIDIA_NIM)) { + return 'nvidia-nim' + } + return 'firstParty' + default: + if ( + ['local', 'openai-compatible'].includes( + getTransportKindForRoute(activeRouteId) ?? '', + ) + ) { + return 'openai' + } + return 'firstParty' + } +} + +function isGithubNativeAnthropicModeForCacheBreak( + env: NodeJS.ProcessEnv, + model: string, +): boolean { + if (!isCacheBreakEnvTruthy(env.CLAUDE_CODE_USE_GITHUB)) return false + const resolvedModel = model.trim() || env.OPENAI_MODEL?.trim() || '' + return resolvedModel.toLowerCase().includes('claude-') +} + +function isCodexCacheBreakRoute( + env: NodeJS.ProcessEnv, + model: string, +): boolean { + const baseUrl = + getNonEmptyEnvValue(env.OPENAI_BASE_URL) ?? + getNonEmptyEnvValue(env.OPENAI_API_BASE) + if (baseUrl?.toLowerCase().includes('/backend-api/codex')) { + return true + } + if (baseUrl?.trim()) { + return false + } + const modelName = (env.OPENAI_MODEL?.trim() || model.trim()).toLowerCase() + return modelName.includes('codex') +} + /** Extended tracking snapshot — everything that could affect the server-side * cache key that we can observe from the client. All fields are optional so * the call site can add incrementally; undefined fields compare as stable. */ @@ -491,6 +758,12 @@ export async function checkResponseForCacheBreak( return } + const { + cacheProvider, + cacheMetricsReliability, + providerRoute, + } = getPromptCacheBreakProviderMetadata(state.model) + // Build explanation from pending changes (if any) const parts: string[] = [] if (changes) { @@ -570,6 +843,13 @@ export async function checkResponseForCacheBreak( timeSinceLastAssistantMsg !== null && timeSinceLastAssistantMsg > CACHE_TTL_1HOUR_MS + const classification = classifyPromptCacheBreak({ + changes, + cacheMetricsReliability, + lastAssistantMsgOver5minAgo, + timeSinceLastAssistantMsg, + }) + // Post PR #19823 BQ analysis (bq-queries/prompt-caching/cache_break_pr19823_analysis.sql): // when all client-side flags are false and the gap is under TTL, ~90% of breaks // are server-side routing/eviction or billed/inference disagreement. Label @@ -577,17 +857,36 @@ export async function checkResponseForCacheBreak( let reason: string if (parts.length > 0) { reason = parts.join(', ') + } else if (classification.kind === 'metrics_unavailable') { + reason = `cache metrics unavailable or unsupported for provider (${cacheProvider})` } else if (lastAssistantMsgOver1hAgo) { reason = 'possible 1h TTL expiry (prompt unchanged)' } else if (lastAssistantMsgOver5minAgo) { reason = 'possible 5min TTL expiry (prompt unchanged)' } else if (timeSinceLastAssistantMsg !== null) { - reason = 'likely server-side (prompt unchanged, <5min gap)' + reason = + cacheMetricsReliability === 'advisory' + ? 'provider-side cache metric instability (prompt unchanged, <5min gap, advisory metrics)' + : 'provider-side cache instability (prompt unchanged, <5min gap)' } else { - reason = 'unknown cause' + reason = 'unknown local mutation or incomplete prompt state' } logEvent('tengu_prompt_cache_break', { + classification: + classification.kind as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + severity: + classification.severity as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + cacheMetricsReliability: + cacheMetricsReliability as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + cacheProvider: + cacheProvider as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + providerRoute: + providerRoute as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + querySource: + querySource as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, + model: + state.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, systemPromptChanged: changes?.systemPromptChanged ?? false, toolSchemasChanged: changes?.toolSchemasChanged ?? false, modelChanged: changes?.modelChanged ?? false, @@ -636,6 +935,7 @@ export async function checkResponseForCacheBreak( prevCacheReadTokens: prevCacheRead, cacheReadTokens, cacheCreationTokens, + tokenDrop, timeSinceLastAssistantMsg: timeSinceLastAssistantMsg ?? -1, lastAssistantMsgOver5minAgo, lastAssistantMsgOver1hAgo, @@ -655,9 +955,9 @@ export async function checkResponseForCacheBreak( } const diffSuffix = diffPath ? `, diff: ${diffPath}` : '' - const summary = `[PROMPT CACHE BREAK] ${reason} [source=${querySource}, call #${state.callCount}, cache read: ${prevCacheRead} → ${cacheReadTokens}, creation: ${cacheCreationTokens}${diffSuffix}]` + const summary = `[PROMPT CACHE BREAK] ${reason} [classification=${classification.kind}, severity=${classification.severity}, reliability=${cacheMetricsReliability}, provider=${cacheProvider}, route=${providerRoute || 'unknown'}, source=${querySource}, call #${state.callCount}, cache read: ${prevCacheRead} → ${cacheReadTokens}, drop: ${tokenDrop}, creation: ${cacheCreationTokens}${diffSuffix}]` - logForDebugging(summary, { level: 'warn' }) + logForDebugging(summary, { level: classification.debugLevel }) state.pendingChanges = null } catch (e: unknown) {