mirror of
https://github.com/Gitlawb/openclaude.git
synced 2026-08-24 02:34:15 -05:00
feat(cache): classify prompt-cache breaks by reliability (#1693)
* feat(cache): classify prompt-cache breaks by reliability * fix(cache): stabilize prompt-cache break metadata detection * fix(cache): honor legacy OpenAI base fallback * fix(cache): normalize OpenAI base URL hints * fix(cache): align cache-break provider flag truthiness * fix(cache): ignore undefined OpenAI base hints * fix(cache): sanitize prompt cache route labels
This commit is contained in:
@@ -6,6 +6,7 @@ import {
|
||||
formatCacheMetricsCompact,
|
||||
formatCacheMetricsFull,
|
||||
addCacheMetrics,
|
||||
getCacheMetricsReliability,
|
||||
} from './cacheMetrics.js'
|
||||
|
||||
describe('extractCacheMetrics — Anthropic (firstParty/bedrock/vertex/foundry)', () => {
|
||||
@@ -447,6 +448,33 @@ describe('resolveCacheProvider — .localhost TLD (RFC 6761)', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('getCacheMetricsReliability', () => {
|
||||
test('Anthropic-native cache metrics are reliable', () => {
|
||||
expect(getCacheMetricsReliability('anthropic')).toBe('reliable')
|
||||
expect(getCacheMetricsReliability('copilot-claude')).toBe('reliable')
|
||||
})
|
||||
|
||||
test('OpenAI-compatible cache metrics are advisory by default', () => {
|
||||
expect(
|
||||
getCacheMetricsReliability(
|
||||
resolveCacheProvider('openai', {
|
||||
openAiBaseUrl: 'https://api.openai.com/v1',
|
||||
}),
|
||||
),
|
||||
).toBe('advisory')
|
||||
expect(getCacheMetricsReliability('codex')).toBe('advisory')
|
||||
expect(getCacheMetricsReliability('kimi')).toBe('advisory')
|
||||
expect(getCacheMetricsReliability('deepseek')).toBe('advisory')
|
||||
expect(getCacheMetricsReliability('gemini')).toBe('advisory')
|
||||
expect(getCacheMetricsReliability('self-hosted')).toBe('advisory')
|
||||
})
|
||||
|
||||
test('providers without cache metric support remain unsupported', () => {
|
||||
expect(getCacheMetricsReliability('copilot')).toBe('unsupported')
|
||||
expect(getCacheMetricsReliability('ollama')).toBe('unsupported')
|
||||
})
|
||||
})
|
||||
|
||||
describe('extractCacheMetrics — hit rate clamp', () => {
|
||||
test('hitRate is clamped to 1.0 on pathological input (read > total)', () => {
|
||||
// Defensive guard: with valid non-negative inputs the math enforces
|
||||
|
||||
@@ -65,6 +65,11 @@ export type CacheAwareProvider =
|
||||
| 'copilot'
|
||||
| 'copilot-claude'
|
||||
|
||||
export type CacheMetricsReliability =
|
||||
| 'reliable'
|
||||
| 'advisory'
|
||||
| 'unsupported'
|
||||
|
||||
/** Unified cache metrics for one API response. */
|
||||
export type CacheMetrics = {
|
||||
/** Tokens served from cache on this request. */
|
||||
@@ -273,6 +278,18 @@ export function resolveCacheProvider(
|
||||
return 'openai'
|
||||
}
|
||||
|
||||
export function getCacheMetricsReliability(
|
||||
provider: CacheAwareProvider,
|
||||
): CacheMetricsReliability {
|
||||
if (provider === 'copilot' || provider === 'ollama') {
|
||||
return 'unsupported'
|
||||
}
|
||||
if (provider === 'anthropic' || provider === 'copilot-claude') {
|
||||
return 'reliable'
|
||||
}
|
||||
return 'advisory'
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the cached-tokens count from a RAW provider usage object, handling
|
||||
* every shape we know about. Callers are the shim layer (openaiShim,
|
||||
|
||||
@@ -0,0 +1,575 @@
|
||||
import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'
|
||||
import type { BetaToolUnion } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs'
|
||||
import type { TextBlockParam } from '@anthropic-ai/sdk/resources/index.mjs'
|
||||
import type { QuerySource } from '../../constants/querySource.js'
|
||||
import type { Message } from '../../types/message.js'
|
||||
import type { DebugLogLevel } from '../../utils/debug.js'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../../test/sharedMutationLock.js'
|
||||
|
||||
type PromptCacheBreakModule = typeof import('./promptCacheBreakDetection.js')
|
||||
type EventCall = {
|
||||
name: string
|
||||
metadata: Record<string, unknown>
|
||||
}
|
||||
type DebugCall = {
|
||||
message: string
|
||||
options?: { level?: DebugLogLevel }
|
||||
}
|
||||
|
||||
const events: EventCall[] = []
|
||||
const debugCalls: DebugCall[] = []
|
||||
|
||||
const logEventMock = mock((name: string, metadata: Record<string, unknown>) => {
|
||||
events.push({ name, metadata })
|
||||
})
|
||||
const logForDebuggingMock = mock(
|
||||
(message: string, options?: { level?: DebugLogLevel }) => {
|
||||
debugCalls.push({ message, options })
|
||||
},
|
||||
)
|
||||
const actualDebugModule = await import('../../utils/debug.js')
|
||||
|
||||
mock.module('../analytics/index.js', () => ({
|
||||
logEvent: logEventMock,
|
||||
}))
|
||||
|
||||
mock.module('src/utils/debug.js', () => ({
|
||||
...actualDebugModule,
|
||||
logForDebugging: logForDebuggingMock,
|
||||
}))
|
||||
|
||||
mock.module('../../utils/debug.js', () => ({
|
||||
...actualDebugModule,
|
||||
logForDebugging: logForDebuggingMock,
|
||||
}))
|
||||
|
||||
const PROVIDER_ENV_KEYS = [
|
||||
'CLAUDE_CODE_USE_OPENAI',
|
||||
'CLAUDE_CODE_USE_GITHUB',
|
||||
'CLAUDE_CODE_USE_GEMINI',
|
||||
'CLAUDE_CODE_USE_MISTRAL',
|
||||
'CLAUDE_CODE_USE_BEDROCK',
|
||||
'CLAUDE_CODE_USE_VERTEX',
|
||||
'CLAUDE_CODE_USE_FOUNDRY',
|
||||
'OPENAI_BASE_URL',
|
||||
'OPENAI_API_BASE',
|
||||
'OPENAI_MODEL',
|
||||
] as const
|
||||
|
||||
const originalEnv: Record<string, string | undefined> = {}
|
||||
let detector: PromptCacheBreakModule | undefined
|
||||
|
||||
for (const key of PROVIDER_ENV_KEYS) {
|
||||
originalEnv[key] = process.env[key]
|
||||
}
|
||||
|
||||
async function loadDetector(): Promise<PromptCacheBreakModule> {
|
||||
detector ??= await import('./promptCacheBreakDetection.js')
|
||||
return detector
|
||||
}
|
||||
|
||||
function restoreProviderEnv(): void {
|
||||
for (const key of PROVIDER_ENV_KEYS) {
|
||||
const value = originalEnv[key]
|
||||
if (value === undefined) {
|
||||
delete process.env[key]
|
||||
} else {
|
||||
process.env[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function clearProviderEnv(): void {
|
||||
for (const key of PROVIDER_ENV_KEYS) {
|
||||
delete process.env[key]
|
||||
}
|
||||
}
|
||||
|
||||
function useOpenAIProvider(): void {
|
||||
clearProviderEnv()
|
||||
process.env.CLAUDE_CODE_USE_OPENAI = '1'
|
||||
process.env.OPENAI_BASE_URL = 'https://api.openai.com/v1'
|
||||
process.env.OPENAI_MODEL = 'gpt-5.5'
|
||||
}
|
||||
|
||||
function useOpenAIProviderWithDisabledFoundryFlag(flagValue: string): void {
|
||||
useOpenAIProvider()
|
||||
process.env.CLAUDE_CODE_USE_FOUNDRY = flagValue
|
||||
}
|
||||
|
||||
function useOpenAIProviderWithWhitespaceBaseFallback(): void {
|
||||
clearProviderEnv()
|
||||
process.env.CLAUDE_CODE_USE_OPENAI = '1'
|
||||
process.env.OPENAI_BASE_URL = ' '
|
||||
process.env.OPENAI_API_BASE = 'https://api.deepseek.com/v1'
|
||||
process.env.OPENAI_MODEL = 'deepseek-chat'
|
||||
}
|
||||
|
||||
function useCodexAliasWithLiteralUndefinedBaseUrl(): void {
|
||||
clearProviderEnv()
|
||||
process.env.CLAUDE_CODE_USE_OPENAI = '1'
|
||||
process.env.OPENAI_BASE_URL = 'undefined'
|
||||
process.env.OPENAI_MODEL = 'codexplan'
|
||||
}
|
||||
|
||||
function useOpenRouterProvider(): void {
|
||||
clearProviderEnv()
|
||||
process.env.CLAUDE_CODE_USE_OPENAI = '1'
|
||||
process.env.OPENAI_BASE_URL = 'https://openrouter.ai/api/v1'
|
||||
process.env.OPENAI_MODEL = 'gpt-4o'
|
||||
}
|
||||
|
||||
function useUnsupportedGithubProvider(): void {
|
||||
clearProviderEnv()
|
||||
process.env.CLAUDE_CODE_USE_GITHUB = '1'
|
||||
process.env.OPENAI_MODEL = 'gpt-4o'
|
||||
}
|
||||
|
||||
function useFoundryProvider(): void {
|
||||
clearProviderEnv()
|
||||
process.env.CLAUDE_CODE_USE_FOUNDRY = '1'
|
||||
}
|
||||
|
||||
function systemBlock(
|
||||
text: string,
|
||||
cacheControl?: Record<string, unknown>,
|
||||
): TextBlockParam[] {
|
||||
return [
|
||||
{
|
||||
type: 'text',
|
||||
text,
|
||||
...(cacheControl ? { cache_control: cacheControl } : {}),
|
||||
},
|
||||
] as TextBlockParam[]
|
||||
}
|
||||
|
||||
function tool(
|
||||
name: string,
|
||||
schema: Record<string, unknown> = {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
},
|
||||
): BetaToolUnion {
|
||||
return {
|
||||
type: 'custom',
|
||||
name,
|
||||
description: `${name} test tool`,
|
||||
input_schema: schema,
|
||||
} as unknown as BetaToolUnion
|
||||
}
|
||||
|
||||
function assistantMessages(gapMs: number): Message[] {
|
||||
return [
|
||||
{
|
||||
type: 'assistant',
|
||||
timestamp: new Date(Date.now() - gapMs).toISOString(),
|
||||
uuid: '00000000-0000-4000-8000-000000000001',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: [],
|
||||
id: 'msg_1',
|
||||
model: 'claude-sonnet-4',
|
||||
usage: {
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
] as unknown as Message[]
|
||||
}
|
||||
|
||||
function snapshot(
|
||||
overrides: Partial<Parameters<PromptCacheBreakModule['recordPromptState']>[0]> = {},
|
||||
): Parameters<PromptCacheBreakModule['recordPromptState']>[0] {
|
||||
return {
|
||||
system: systemBlock('base system prompt'),
|
||||
toolSchemas: [tool('Read')],
|
||||
querySource: 'repl_main_thread' as QuerySource,
|
||||
model: 'claude-sonnet-4',
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
async function triggerCacheDrop({
|
||||
first = snapshot(),
|
||||
second = snapshot(),
|
||||
secondMessages = assistantMessages(60_000),
|
||||
}: {
|
||||
first?: Parameters<PromptCacheBreakModule['recordPromptState']>[0]
|
||||
second?: Parameters<PromptCacheBreakModule['recordPromptState']>[0]
|
||||
secondMessages?: Message[]
|
||||
} = {}): Promise<EventCall> {
|
||||
const mod = await loadDetector()
|
||||
mod.recordPromptState(first)
|
||||
await mod.checkResponseForCacheBreak(
|
||||
first.querySource,
|
||||
10_000,
|
||||
0,
|
||||
assistantMessages(60_000),
|
||||
first.agentId,
|
||||
'req-prev',
|
||||
)
|
||||
|
||||
mod.recordPromptState(second)
|
||||
await mod.checkResponseForCacheBreak(
|
||||
second.querySource,
|
||||
1_000,
|
||||
0,
|
||||
secondMessages,
|
||||
second.agentId,
|
||||
'req-break',
|
||||
)
|
||||
|
||||
const event = events.findLast(e => e.name === 'tengu_prompt_cache_break')
|
||||
expect(event).toBeDefined()
|
||||
return event!
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock('promptCacheBreakDetection.test.ts')
|
||||
events.length = 0
|
||||
debugCalls.length = 0
|
||||
logEventMock.mockClear()
|
||||
logForDebuggingMock.mockClear()
|
||||
clearProviderEnv()
|
||||
const mod = await loadDetector()
|
||||
mod.resetPromptCacheBreakDetection()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
try {
|
||||
detector?.resetPromptCacheBreakDetection()
|
||||
restoreProviderEnv()
|
||||
} finally {
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
describe('prompt cache break taxonomy', () => {
|
||||
test('tool addition/removal classifies as expected tool schema change', async () => {
|
||||
const event = await triggerCacheDrop({
|
||||
first: snapshot({ toolSchemas: [tool('Read'), tool('Edit')] }),
|
||||
second: snapshot({ toolSchemas: [tool('Edit'), tool('Bash')] }),
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'expected_tool_schema_change',
|
||||
addedToolCount: 1,
|
||||
removedToolCount: 1,
|
||||
addedTools: 'Bash',
|
||||
removedTools: 'Read',
|
||||
})
|
||||
})
|
||||
|
||||
test('changed MCP tool schema names are sanitized in analytics', async () => {
|
||||
const event = await triggerCacheDrop({
|
||||
first: snapshot({
|
||||
toolSchemas: [
|
||||
tool('mcp__repo_server__search', {
|
||||
type: 'object',
|
||||
properties: { query: { type: 'string' } },
|
||||
}),
|
||||
],
|
||||
}),
|
||||
second: snapshot({
|
||||
toolSchemas: [
|
||||
tool('mcp__repo_server__search', {
|
||||
type: 'object',
|
||||
properties: { query: { type: 'string' }, limit: { type: 'number' } },
|
||||
}),
|
||||
],
|
||||
}),
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'expected_tool_schema_change',
|
||||
changedToolSchemas: 'mcp',
|
||||
})
|
||||
})
|
||||
|
||||
test('system prompt hash change classifies as expected local prompt change', async () => {
|
||||
const event = await triggerCacheDrop({
|
||||
first: snapshot({ system: systemBlock('base system prompt') }),
|
||||
second: snapshot({ system: systemBlock('updated system prompt') }),
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'expected_local_prompt_change',
|
||||
severity: 'info',
|
||||
})
|
||||
})
|
||||
|
||||
test('cache-control-only change classifies separately', async () => {
|
||||
const event = await triggerCacheDrop({
|
||||
first: snapshot({
|
||||
system: systemBlock('base system prompt', { type: 'ephemeral' }),
|
||||
}),
|
||||
second: snapshot({
|
||||
system: systemBlock('base system prompt', {
|
||||
type: 'ephemeral',
|
||||
ttl: '1h',
|
||||
}),
|
||||
}),
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'expected_cache_control_change',
|
||||
cacheControlChanged: true,
|
||||
systemPromptChanged: false,
|
||||
})
|
||||
})
|
||||
|
||||
test('global cache strategy change classifies with cache-control changes', async () => {
|
||||
const event = await triggerCacheDrop({
|
||||
first: snapshot({ globalCacheStrategy: 'tool_based' }),
|
||||
second: snapshot({ globalCacheStrategy: 'system_prompt' }),
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'expected_cache_control_change',
|
||||
cacheControlChanged: false,
|
||||
globalCacheStrategyChanged: true,
|
||||
prevGlobalCacheStrategy: 'tool_based',
|
||||
newGlobalCacheStrategy: 'system_prompt',
|
||||
})
|
||||
})
|
||||
|
||||
test('model, beta, effort, and extra-body changes classify as expected model or beta change', async () => {
|
||||
const event = await triggerCacheDrop({
|
||||
first: snapshot({
|
||||
model: 'claude-sonnet-4',
|
||||
betas: ['beta-a'],
|
||||
effortValue: 'medium',
|
||||
extraBodyParams: { metadata: { source: 'first' } },
|
||||
}),
|
||||
second: snapshot({
|
||||
model: 'claude-opus-4',
|
||||
betas: ['beta-b'],
|
||||
effortValue: 'high',
|
||||
extraBodyParams: { metadata: { source: 'second' } },
|
||||
}),
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'expected_model_or_beta_change',
|
||||
modelChanged: true,
|
||||
betasChanged: true,
|
||||
effortChanged: true,
|
||||
extraBodyChanged: true,
|
||||
addedBetas: 'beta-b',
|
||||
removedBetas: 'beta-a',
|
||||
})
|
||||
})
|
||||
|
||||
test('same prompt/schema under TTL on advisory provider classifies as provider cache instability with info severity', async () => {
|
||||
useOpenAIProvider()
|
||||
|
||||
const event = await triggerCacheDrop()
|
||||
const debug = debugCalls.findLast(call =>
|
||||
call.message.startsWith('[PROMPT CACHE BREAK]'),
|
||||
)
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'provider_cache_instability',
|
||||
cacheMetricsReliability: 'advisory',
|
||||
severity: 'info',
|
||||
prevCacheReadTokens: 10_000,
|
||||
cacheReadTokens: 1_000,
|
||||
tokenDrop: 9_000,
|
||||
querySource: 'repl_main_thread',
|
||||
model: 'claude-sonnet-4',
|
||||
providerRoute: 'openai',
|
||||
requestId: 'req-break',
|
||||
})
|
||||
expect(debug?.options).toEqual({ level: 'info' })
|
||||
expect(debug?.message).not.toMatch(/app crash|crash|local mutation/i)
|
||||
})
|
||||
|
||||
for (const falseyFoundryFlag of ['off', 'no'] as const) {
|
||||
test(`stale Foundry=${falseyFoundryFlag} env flag does not override OpenAI cache metadata`, async () => {
|
||||
useOpenAIProviderWithDisabledFoundryFlag(falseyFoundryFlag)
|
||||
|
||||
const event = await triggerCacheDrop()
|
||||
const debug = debugCalls.findLast(call =>
|
||||
call.message.startsWith('[PROMPT CACHE BREAK]'),
|
||||
)
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'provider_cache_instability',
|
||||
cacheMetricsReliability: 'advisory',
|
||||
cacheProvider: 'openai',
|
||||
providerRoute: 'openai',
|
||||
severity: 'info',
|
||||
})
|
||||
expect(debug?.options).toEqual({ level: 'info' })
|
||||
})
|
||||
}
|
||||
|
||||
test('blank OpenAI base URL falls back to legacy API base for cache provider metadata', async () => {
|
||||
useOpenAIProviderWithWhitespaceBaseFallback()
|
||||
|
||||
const event = await triggerCacheDrop({
|
||||
first: snapshot({ model: 'deepseek-chat' }),
|
||||
second: snapshot({ model: 'deepseek-chat' }),
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'provider_cache_instability',
|
||||
cacheMetricsReliability: 'advisory',
|
||||
cacheProvider: 'deepseek',
|
||||
severity: 'info',
|
||||
})
|
||||
})
|
||||
|
||||
test('literal undefined OpenAI base URL still reports Codex alias metadata', async () => {
|
||||
useCodexAliasWithLiteralUndefinedBaseUrl()
|
||||
|
||||
const event = await triggerCacheDrop({
|
||||
first: snapshot({ model: 'codexplan' }),
|
||||
second: snapshot({ model: 'codexplan' }),
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'provider_cache_instability',
|
||||
cacheMetricsReliability: 'advisory',
|
||||
cacheProvider: 'codex',
|
||||
providerRoute: 'codex',
|
||||
severity: 'info',
|
||||
})
|
||||
})
|
||||
|
||||
test('descriptor OpenAI-compatible route IDs are normalized before logging', async () => {
|
||||
useOpenRouterProvider()
|
||||
|
||||
const event = await triggerCacheDrop({
|
||||
first: snapshot({ model: 'gpt-4o' }),
|
||||
second: snapshot({ model: 'gpt-4o' }),
|
||||
})
|
||||
const debug = debugCalls.findLast(call =>
|
||||
call.message.startsWith('[PROMPT CACHE BREAK]'),
|
||||
)
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'provider_cache_instability',
|
||||
cacheMetricsReliability: 'advisory',
|
||||
cacheProvider: 'openai',
|
||||
providerRoute: 'openai-compatible',
|
||||
severity: 'info',
|
||||
})
|
||||
expect(debug?.message).toContain('route=openai-compatible')
|
||||
expect(debug?.message).not.toContain('openrouter')
|
||||
})
|
||||
|
||||
test('TTL-window cache drops classify as possible TTL expiry', async () => {
|
||||
const event = await triggerCacheDrop({
|
||||
secondMessages: assistantMessages(61 * 60 * 1000),
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'possible_ttl_expiry',
|
||||
lastAssistantMsgOver5minAgo: true,
|
||||
lastAssistantMsgOver1hAgo: true,
|
||||
})
|
||||
})
|
||||
|
||||
test('unknown or incomplete state classifies as unknown local mutation, not provider-side instability', async () => {
|
||||
const event = await triggerCacheDrop({
|
||||
secondMessages: [],
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'unknown_local_mutation',
|
||||
timeSinceLastAssistantMsg: -1,
|
||||
})
|
||||
})
|
||||
|
||||
test('unsupported provider metrics classify as metrics unavailable', async () => {
|
||||
useUnsupportedGithubProvider()
|
||||
|
||||
const event = await triggerCacheDrop({
|
||||
first: snapshot({ model: 'gpt-4o' }),
|
||||
second: snapshot({ model: 'gpt-4o' }),
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'metrics_unavailable',
|
||||
cacheMetricsReliability: 'unsupported',
|
||||
providerRoute: 'github',
|
||||
})
|
||||
})
|
||||
|
||||
test('unsupported provider metrics do not hide known local prompt changes', async () => {
|
||||
useUnsupportedGithubProvider()
|
||||
|
||||
const event = await triggerCacheDrop({
|
||||
first: snapshot({
|
||||
model: 'gpt-4o',
|
||||
system: systemBlock('base system prompt'),
|
||||
}),
|
||||
second: snapshot({
|
||||
model: 'gpt-4o',
|
||||
system: systemBlock('changed system prompt'),
|
||||
}),
|
||||
})
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'expected_local_prompt_change',
|
||||
cacheMetricsReliability: 'unsupported',
|
||||
systemPromptChanged: true,
|
||||
})
|
||||
})
|
||||
|
||||
test('legacy Foundry provider does not get mislabeled as anthropic route', async () => {
|
||||
useFoundryProvider()
|
||||
|
||||
const event = await triggerCacheDrop()
|
||||
const debug = debugCalls.findLast(call =>
|
||||
call.message.startsWith('[PROMPT CACHE BREAK]'),
|
||||
)
|
||||
|
||||
expect(event.metadata).toMatchObject({
|
||||
classification: 'provider_cache_instability',
|
||||
cacheMetricsReliability: 'reliable',
|
||||
cacheProvider: 'anthropic',
|
||||
providerRoute: 'foundry',
|
||||
severity: 'warning',
|
||||
})
|
||||
expect(debug?.options).toEqual({ level: 'warn' })
|
||||
})
|
||||
|
||||
test('cache deletion expected drops remain suppressed', async () => {
|
||||
const mod = await loadDetector()
|
||||
const first = snapshot()
|
||||
|
||||
mod.recordPromptState(first)
|
||||
await mod.checkResponseForCacheBreak(
|
||||
first.querySource,
|
||||
10_000,
|
||||
0,
|
||||
assistantMessages(60_000),
|
||||
first.agentId,
|
||||
'req-prev',
|
||||
)
|
||||
|
||||
mod.recordPromptState(snapshot())
|
||||
mod.notifyCacheDeletion(first.querySource, first.agentId)
|
||||
await mod.checkResponseForCacheBreak(
|
||||
first.querySource,
|
||||
1_000,
|
||||
0,
|
||||
assistantMessages(60_000),
|
||||
first.agentId,
|
||||
'req-drop',
|
||||
)
|
||||
|
||||
expect(events.filter(e => e.name === 'tengu_prompt_cache_break')).toEqual([])
|
||||
expect(
|
||||
debugCalls.some(call => call.message.includes('cache deletion applied')),
|
||||
).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -5,16 +5,25 @@ import { mkdir, writeFile } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import type { AgentId } from 'src/types/ids.js'
|
||||
import type { Message } from 'src/types/message.js'
|
||||
import { logForDebugging } from 'src/utils/debug.js'
|
||||
import { type DebugLogLevel, logForDebugging } from 'src/utils/debug.js'
|
||||
import { djb2Hash } from 'src/utils/hash.js'
|
||||
import { logError } from 'src/utils/log.js'
|
||||
import type { APIProvider } from 'src/utils/model/providers.js'
|
||||
import { getClaudeTempDir } from 'src/utils/permissions/filesystem.js'
|
||||
import { jsonStringify } from 'src/utils/slowOperations.js'
|
||||
import type { QuerySource } from '../../constants/querySource.js'
|
||||
import {
|
||||
getTransportKindForRoute,
|
||||
resolveActiveRouteIdFromEnv,
|
||||
} from '../../integrations/routeMetadata.js'
|
||||
import {
|
||||
type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
logEvent,
|
||||
} from '../analytics/index.js'
|
||||
import {
|
||||
getCacheMetricsReliability,
|
||||
resolveCacheProvider,
|
||||
type CacheMetricsReliability,
|
||||
} from './cacheMetrics.js'
|
||||
|
||||
function getCacheBreakDiffPath(): string {
|
||||
const chars = 'abcdefghijklmnopqrstuvwxyz0123456789'
|
||||
@@ -98,6 +107,24 @@ type PendingChanges = {
|
||||
buildPrevDiffableContent: () => string
|
||||
}
|
||||
|
||||
export type PromptCacheBreakKind =
|
||||
| 'expected_local_prompt_change'
|
||||
| 'expected_tool_schema_change'
|
||||
| 'expected_cache_control_change'
|
||||
| 'expected_model_or_beta_change'
|
||||
| 'provider_cache_instability'
|
||||
| 'possible_ttl_expiry'
|
||||
| 'unknown_local_mutation'
|
||||
| 'metrics_unavailable'
|
||||
|
||||
type PromptCacheBreakSeverity = 'debug' | 'info' | 'warning'
|
||||
|
||||
type PromptCacheBreakClassification = {
|
||||
kind: PromptCacheBreakKind
|
||||
severity: PromptCacheBreakSeverity
|
||||
debugLevel: DebugLogLevel
|
||||
}
|
||||
|
||||
const previousStateBySource = new Map<string, PreviousState>()
|
||||
|
||||
// Cap the number of tracked sources to prevent unbounded memory growth.
|
||||
@@ -168,7 +195,7 @@ function stripCacheControl(
|
||||
}
|
||||
|
||||
function computeHash(data: unknown): number {
|
||||
const str = jsonStringify(data)
|
||||
const str = stringifyCacheBreakData(data)
|
||||
if (typeof Bun !== 'undefined') {
|
||||
const hash = Bun.hash(str)
|
||||
// Bun.hash can return bigint for large inputs; convert to number safely
|
||||
@@ -178,6 +205,40 @@ function computeHash(data: unknown): number {
|
||||
return djb2Hash(str)
|
||||
}
|
||||
|
||||
function stringifyCacheBreakData(data: unknown): string {
|
||||
return JSON.stringify(data) ?? ''
|
||||
}
|
||||
|
||||
function isCacheBreakEnvTruthy(value: string | undefined): boolean {
|
||||
const normalized = value?.trim().toLowerCase()
|
||||
return (
|
||||
normalized === '1' ||
|
||||
normalized === 'true' ||
|
||||
normalized === 'yes' ||
|
||||
normalized === 'on'
|
||||
)
|
||||
}
|
||||
|
||||
function getNonEmptyEnvValue(value: string | undefined): string | undefined {
|
||||
const trimmed = value?.trim()
|
||||
return trimmed && trimmed !== 'undefined' ? trimmed : undefined
|
||||
}
|
||||
|
||||
const SAFE_PROMPT_CACHE_PROVIDER_ROUTES = new Set<string>([
|
||||
'anthropic',
|
||||
'openai',
|
||||
'custom',
|
||||
'gemini',
|
||||
'mistral',
|
||||
'github',
|
||||
'bedrock',
|
||||
'vertex',
|
||||
'nvidia-nim',
|
||||
'minimax',
|
||||
'xiaomi-mimo',
|
||||
'xai',
|
||||
])
|
||||
|
||||
/** MCP tool names are user-controlled (server config) and may leak filepaths.
|
||||
* Collapse them to 'mcp'; built-in names are a fixed vocabulary. */
|
||||
function sanitizeToolName(name: string): string {
|
||||
@@ -213,7 +274,8 @@ function buildDiffableContent(
|
||||
.map(t => {
|
||||
if (!('name' in t)) return 'unknown'
|
||||
const desc = 'description' in t ? t.description : ''
|
||||
const schema = 'input_schema' in t ? jsonStringify(t.input_schema) : ''
|
||||
const schema =
|
||||
'input_schema' in t ? stringifyCacheBreakData(t.input_schema) : ''
|
||||
return `${t.name}\n description: ${desc}\n input_schema: ${schema}`
|
||||
})
|
||||
.sort()
|
||||
@@ -221,6 +283,211 @@ function buildDiffableContent(
|
||||
return `Model: ${model}\n\n=== System Prompt ===\n\n${systemText}\n\n=== Tools (${tools.length}) ===\n\n${toolDetails}\n`
|
||||
}
|
||||
|
||||
function classifyPromptCacheBreak({
|
||||
changes,
|
||||
cacheMetricsReliability,
|
||||
lastAssistantMsgOver5minAgo,
|
||||
timeSinceLastAssistantMsg,
|
||||
}: {
|
||||
changes: PendingChanges | null
|
||||
cacheMetricsReliability: CacheMetricsReliability
|
||||
lastAssistantMsgOver5minAgo: boolean
|
||||
timeSinceLastAssistantMsg: number | null
|
||||
}): PromptCacheBreakClassification {
|
||||
if (changes) {
|
||||
if (changes.systemPromptChanged) {
|
||||
return {
|
||||
kind: 'expected_local_prompt_change',
|
||||
severity: 'info',
|
||||
debugLevel: 'info',
|
||||
}
|
||||
}
|
||||
if (changes.toolSchemasChanged) {
|
||||
return {
|
||||
kind: 'expected_tool_schema_change',
|
||||
severity: 'info',
|
||||
debugLevel: 'info',
|
||||
}
|
||||
}
|
||||
if (
|
||||
(changes.cacheControlChanged || changes.globalCacheStrategyChanged) &&
|
||||
!changes.systemPromptChanged
|
||||
) {
|
||||
return {
|
||||
kind: 'expected_cache_control_change',
|
||||
severity: 'info',
|
||||
debugLevel: 'info',
|
||||
}
|
||||
}
|
||||
if (
|
||||
changes.modelChanged ||
|
||||
changes.fastModeChanged ||
|
||||
changes.betasChanged ||
|
||||
changes.autoModeChanged ||
|
||||
changes.overageChanged ||
|
||||
changes.cachedMCChanged ||
|
||||
changes.effortChanged ||
|
||||
changes.extraBodyChanged
|
||||
) {
|
||||
return {
|
||||
kind: 'expected_model_or_beta_change',
|
||||
severity: 'info',
|
||||
debugLevel: 'info',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (cacheMetricsReliability === 'unsupported') {
|
||||
return {
|
||||
kind: 'metrics_unavailable',
|
||||
severity: 'info',
|
||||
debugLevel: 'info',
|
||||
}
|
||||
}
|
||||
|
||||
if (lastAssistantMsgOver5minAgo) {
|
||||
return {
|
||||
kind: 'possible_ttl_expiry',
|
||||
severity: 'info',
|
||||
debugLevel: 'info',
|
||||
}
|
||||
}
|
||||
|
||||
if (timeSinceLastAssistantMsg !== null) {
|
||||
const advisory = cacheMetricsReliability === 'advisory'
|
||||
return {
|
||||
kind: 'provider_cache_instability',
|
||||
severity: advisory ? 'info' : 'warning',
|
||||
debugLevel: advisory ? 'info' : 'warn',
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'unknown_local_mutation',
|
||||
severity: 'warning',
|
||||
debugLevel: 'warn',
|
||||
}
|
||||
}
|
||||
|
||||
function getPromptCacheBreakProviderMetadata(model: string): {
|
||||
cacheProvider: string
|
||||
cacheMetricsReliability: CacheMetricsReliability
|
||||
providerRoute: string
|
||||
} {
|
||||
const activeRouteId = resolveActiveRouteIdFromEnv(process.env)
|
||||
const apiProvider = resolvePromptCacheBreakAPIProvider(
|
||||
process.env,
|
||||
activeRouteId,
|
||||
model,
|
||||
)
|
||||
const cacheProvider = resolveCacheProvider(apiProvider, {
|
||||
githubNativeAnthropic: isGithubNativeAnthropicModeForCacheBreak(
|
||||
process.env,
|
||||
model,
|
||||
),
|
||||
openAiBaseUrl:
|
||||
getNonEmptyEnvValue(process.env.OPENAI_BASE_URL) ??
|
||||
getNonEmptyEnvValue(process.env.OPENAI_API_BASE),
|
||||
})
|
||||
return {
|
||||
cacheProvider,
|
||||
cacheMetricsReliability: getCacheMetricsReliability(cacheProvider),
|
||||
providerRoute: getPromptCacheBreakProviderRoute(
|
||||
activeRouteId,
|
||||
apiProvider,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
function getPromptCacheBreakProviderRoute(
|
||||
activeRouteId: string | null,
|
||||
apiProvider: APIProvider,
|
||||
): string {
|
||||
if (apiProvider === 'codex') {
|
||||
return 'codex'
|
||||
}
|
||||
if (activeRouteId === 'anthropic' && apiProvider !== 'firstParty') {
|
||||
return apiProvider
|
||||
}
|
||||
if (!activeRouteId) {
|
||||
return apiProvider
|
||||
}
|
||||
if (SAFE_PROMPT_CACHE_PROVIDER_ROUTES.has(activeRouteId)) {
|
||||
return activeRouteId
|
||||
}
|
||||
return getTransportKindForRoute(activeRouteId) ?? apiProvider
|
||||
}
|
||||
|
||||
function resolvePromptCacheBreakAPIProvider(
|
||||
env: NodeJS.ProcessEnv,
|
||||
activeRouteId: string | null,
|
||||
model: string,
|
||||
): APIProvider {
|
||||
if (isCacheBreakEnvTruthy(env.CLAUDE_CODE_USE_FOUNDRY)) {
|
||||
return 'foundry'
|
||||
}
|
||||
|
||||
switch (activeRouteId) {
|
||||
case 'gemini':
|
||||
case 'mistral':
|
||||
case 'github':
|
||||
case 'bedrock':
|
||||
case 'vertex':
|
||||
case 'nvidia-nim':
|
||||
case 'minimax':
|
||||
case 'xiaomi-mimo':
|
||||
case 'xai':
|
||||
return activeRouteId
|
||||
case 'openai':
|
||||
case 'custom':
|
||||
if (isCacheBreakEnvTruthy(env.NVIDIA_NIM)) {
|
||||
return 'nvidia-nim'
|
||||
}
|
||||
return isCodexCacheBreakRoute(env, model) ? 'codex' : 'openai'
|
||||
case 'anthropic':
|
||||
case null:
|
||||
if (isCacheBreakEnvTruthy(env.NVIDIA_NIM)) {
|
||||
return 'nvidia-nim'
|
||||
}
|
||||
return 'firstParty'
|
||||
default:
|
||||
if (
|
||||
['local', 'openai-compatible'].includes(
|
||||
getTransportKindForRoute(activeRouteId) ?? '',
|
||||
)
|
||||
) {
|
||||
return 'openai'
|
||||
}
|
||||
return 'firstParty'
|
||||
}
|
||||
}
|
||||
|
||||
function isGithubNativeAnthropicModeForCacheBreak(
|
||||
env: NodeJS.ProcessEnv,
|
||||
model: string,
|
||||
): boolean {
|
||||
if (!isCacheBreakEnvTruthy(env.CLAUDE_CODE_USE_GITHUB)) return false
|
||||
const resolvedModel = model.trim() || env.OPENAI_MODEL?.trim() || ''
|
||||
return resolvedModel.toLowerCase().includes('claude-')
|
||||
}
|
||||
|
||||
function isCodexCacheBreakRoute(
|
||||
env: NodeJS.ProcessEnv,
|
||||
model: string,
|
||||
): boolean {
|
||||
const baseUrl =
|
||||
getNonEmptyEnvValue(env.OPENAI_BASE_URL) ??
|
||||
getNonEmptyEnvValue(env.OPENAI_API_BASE)
|
||||
if (baseUrl?.toLowerCase().includes('/backend-api/codex')) {
|
||||
return true
|
||||
}
|
||||
if (baseUrl?.trim()) {
|
||||
return false
|
||||
}
|
||||
const modelName = (env.OPENAI_MODEL?.trim() || model.trim()).toLowerCase()
|
||||
return modelName.includes('codex')
|
||||
}
|
||||
|
||||
/** Extended tracking snapshot — everything that could affect the server-side
|
||||
* cache key that we can observe from the client. All fields are optional so
|
||||
* the call site can add incrementally; undefined fields compare as stable. */
|
||||
@@ -491,6 +758,12 @@ export async function checkResponseForCacheBreak(
|
||||
return
|
||||
}
|
||||
|
||||
const {
|
||||
cacheProvider,
|
||||
cacheMetricsReliability,
|
||||
providerRoute,
|
||||
} = getPromptCacheBreakProviderMetadata(state.model)
|
||||
|
||||
// Build explanation from pending changes (if any)
|
||||
const parts: string[] = []
|
||||
if (changes) {
|
||||
@@ -570,6 +843,13 @@ export async function checkResponseForCacheBreak(
|
||||
timeSinceLastAssistantMsg !== null &&
|
||||
timeSinceLastAssistantMsg > CACHE_TTL_1HOUR_MS
|
||||
|
||||
const classification = classifyPromptCacheBreak({
|
||||
changes,
|
||||
cacheMetricsReliability,
|
||||
lastAssistantMsgOver5minAgo,
|
||||
timeSinceLastAssistantMsg,
|
||||
})
|
||||
|
||||
// Post PR #19823 BQ analysis (bq-queries/prompt-caching/cache_break_pr19823_analysis.sql):
|
||||
// when all client-side flags are false and the gap is under TTL, ~90% of breaks
|
||||
// are server-side routing/eviction or billed/inference disagreement. Label
|
||||
@@ -577,17 +857,36 @@ export async function checkResponseForCacheBreak(
|
||||
let reason: string
|
||||
if (parts.length > 0) {
|
||||
reason = parts.join(', ')
|
||||
} else if (classification.kind === 'metrics_unavailable') {
|
||||
reason = `cache metrics unavailable or unsupported for provider (${cacheProvider})`
|
||||
} else if (lastAssistantMsgOver1hAgo) {
|
||||
reason = 'possible 1h TTL expiry (prompt unchanged)'
|
||||
} else if (lastAssistantMsgOver5minAgo) {
|
||||
reason = 'possible 5min TTL expiry (prompt unchanged)'
|
||||
} else if (timeSinceLastAssistantMsg !== null) {
|
||||
reason = 'likely server-side (prompt unchanged, <5min gap)'
|
||||
reason =
|
||||
cacheMetricsReliability === 'advisory'
|
||||
? 'provider-side cache metric instability (prompt unchanged, <5min gap, advisory metrics)'
|
||||
: 'provider-side cache instability (prompt unchanged, <5min gap)'
|
||||
} else {
|
||||
reason = 'unknown cause'
|
||||
reason = 'unknown local mutation or incomplete prompt state'
|
||||
}
|
||||
|
||||
logEvent('tengu_prompt_cache_break', {
|
||||
classification:
|
||||
classification.kind as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
severity:
|
||||
classification.severity as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
cacheMetricsReliability:
|
||||
cacheMetricsReliability as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
cacheProvider:
|
||||
cacheProvider as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
providerRoute:
|
||||
providerRoute as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
querySource:
|
||||
querySource as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
model:
|
||||
state.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
systemPromptChanged: changes?.systemPromptChanged ?? false,
|
||||
toolSchemasChanged: changes?.toolSchemasChanged ?? false,
|
||||
modelChanged: changes?.modelChanged ?? false,
|
||||
@@ -636,6 +935,7 @@ export async function checkResponseForCacheBreak(
|
||||
prevCacheReadTokens: prevCacheRead,
|
||||
cacheReadTokens,
|
||||
cacheCreationTokens,
|
||||
tokenDrop,
|
||||
timeSinceLastAssistantMsg: timeSinceLastAssistantMsg ?? -1,
|
||||
lastAssistantMsgOver5minAgo,
|
||||
lastAssistantMsgOver1hAgo,
|
||||
@@ -655,9 +955,9 @@ export async function checkResponseForCacheBreak(
|
||||
}
|
||||
|
||||
const diffSuffix = diffPath ? `, diff: ${diffPath}` : ''
|
||||
const summary = `[PROMPT CACHE BREAK] ${reason} [source=${querySource}, call #${state.callCount}, cache read: ${prevCacheRead} → ${cacheReadTokens}, creation: ${cacheCreationTokens}${diffSuffix}]`
|
||||
const summary = `[PROMPT CACHE BREAK] ${reason} [classification=${classification.kind}, severity=${classification.severity}, reliability=${cacheMetricsReliability}, provider=${cacheProvider}, route=${providerRoute || 'unknown'}, source=${querySource}, call #${state.callCount}, cache read: ${prevCacheRead} → ${cacheReadTokens}, drop: ${tokenDrop}, creation: ${cacheCreationTokens}${diffSuffix}]`
|
||||
|
||||
logForDebugging(summary, { level: 'warn' })
|
||||
logForDebugging(summary, { level: classification.debugLevel })
|
||||
|
||||
state.pendingChanges = null
|
||||
} catch (e: unknown) {
|
||||
|
||||
Reference in New Issue
Block a user