feat(cache): classify prompt-cache breaks by reliability (#1693)

* feat(cache): classify prompt-cache breaks by reliability

* fix(cache): stabilize prompt-cache break metadata detection

* fix(cache): honor legacy OpenAI base fallback

* fix(cache): normalize OpenAI base URL hints

* fix(cache): align cache-break provider flag truthiness

* fix(cache): ignore undefined OpenAI base hints

* fix(cache): sanitize prompt cache route labels
This commit is contained in:
Bogdan
2026-06-18 08:42:41 +08:00
committed by GitHub
parent beab67b44a
commit 4cf981200f
4 changed files with 928 additions and 8 deletions
+28
View File
@@ -6,6 +6,7 @@ import {
formatCacheMetricsCompact,
formatCacheMetricsFull,
addCacheMetrics,
getCacheMetricsReliability,
} from './cacheMetrics.js'
describe('extractCacheMetrics — Anthropic (firstParty/bedrock/vertex/foundry)', () => {
@@ -447,6 +448,33 @@ describe('resolveCacheProvider — .localhost TLD (RFC 6761)', () => {
})
})
describe('getCacheMetricsReliability', () => {
test('Anthropic-native cache metrics are reliable', () => {
expect(getCacheMetricsReliability('anthropic')).toBe('reliable')
expect(getCacheMetricsReliability('copilot-claude')).toBe('reliable')
})
test('OpenAI-compatible cache metrics are advisory by default', () => {
expect(
getCacheMetricsReliability(
resolveCacheProvider('openai', {
openAiBaseUrl: 'https://api.openai.com/v1',
}),
),
).toBe('advisory')
expect(getCacheMetricsReliability('codex')).toBe('advisory')
expect(getCacheMetricsReliability('kimi')).toBe('advisory')
expect(getCacheMetricsReliability('deepseek')).toBe('advisory')
expect(getCacheMetricsReliability('gemini')).toBe('advisory')
expect(getCacheMetricsReliability('self-hosted')).toBe('advisory')
})
test('providers without cache metric support remain unsupported', () => {
expect(getCacheMetricsReliability('copilot')).toBe('unsupported')
expect(getCacheMetricsReliability('ollama')).toBe('unsupported')
})
})
describe('extractCacheMetrics — hit rate clamp', () => {
test('hitRate is clamped to 1.0 on pathological input (read > total)', () => {
// Defensive guard: with valid non-negative inputs the math enforces
+17
View File
@@ -65,6 +65,11 @@ export type CacheAwareProvider =
| 'copilot'
| 'copilot-claude'
export type CacheMetricsReliability =
| 'reliable'
| 'advisory'
| 'unsupported'
/** Unified cache metrics for one API response. */
export type CacheMetrics = {
/** Tokens served from cache on this request. */
@@ -273,6 +278,18 @@ export function resolveCacheProvider(
return 'openai'
}
export function getCacheMetricsReliability(
provider: CacheAwareProvider,
): CacheMetricsReliability {
if (provider === 'copilot' || provider === 'ollama') {
return 'unsupported'
}
if (provider === 'anthropic' || provider === 'copilot-claude') {
return 'reliable'
}
return 'advisory'
}
/**
* Read the cached-tokens count from a RAW provider usage object, handling
* every shape we know about. Callers are the shim layer (openaiShim,
@@ -0,0 +1,575 @@
import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'
import type { BetaToolUnion } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs'
import type { TextBlockParam } from '@anthropic-ai/sdk/resources/index.mjs'
import type { QuerySource } from '../../constants/querySource.js'
import type { Message } from '../../types/message.js'
import type { DebugLogLevel } from '../../utils/debug.js'
import {
acquireSharedMutationLock,
releaseSharedMutationLock,
} from '../../test/sharedMutationLock.js'
type PromptCacheBreakModule = typeof import('./promptCacheBreakDetection.js')
type EventCall = {
name: string
metadata: Record<string, unknown>
}
type DebugCall = {
message: string
options?: { level?: DebugLogLevel }
}
const events: EventCall[] = []
const debugCalls: DebugCall[] = []
const logEventMock = mock((name: string, metadata: Record<string, unknown>) => {
events.push({ name, metadata })
})
const logForDebuggingMock = mock(
(message: string, options?: { level?: DebugLogLevel }) => {
debugCalls.push({ message, options })
},
)
const actualDebugModule = await import('../../utils/debug.js')
mock.module('../analytics/index.js', () => ({
logEvent: logEventMock,
}))
mock.module('src/utils/debug.js', () => ({
...actualDebugModule,
logForDebugging: logForDebuggingMock,
}))
mock.module('../../utils/debug.js', () => ({
...actualDebugModule,
logForDebugging: logForDebuggingMock,
}))
const PROVIDER_ENV_KEYS = [
'CLAUDE_CODE_USE_OPENAI',
'CLAUDE_CODE_USE_GITHUB',
'CLAUDE_CODE_USE_GEMINI',
'CLAUDE_CODE_USE_MISTRAL',
'CLAUDE_CODE_USE_BEDROCK',
'CLAUDE_CODE_USE_VERTEX',
'CLAUDE_CODE_USE_FOUNDRY',
'OPENAI_BASE_URL',
'OPENAI_API_BASE',
'OPENAI_MODEL',
] as const
const originalEnv: Record<string, string | undefined> = {}
let detector: PromptCacheBreakModule | undefined
for (const key of PROVIDER_ENV_KEYS) {
originalEnv[key] = process.env[key]
}
async function loadDetector(): Promise<PromptCacheBreakModule> {
detector ??= await import('./promptCacheBreakDetection.js')
return detector
}
function restoreProviderEnv(): void {
for (const key of PROVIDER_ENV_KEYS) {
const value = originalEnv[key]
if (value === undefined) {
delete process.env[key]
} else {
process.env[key] = value
}
}
}
function clearProviderEnv(): void {
for (const key of PROVIDER_ENV_KEYS) {
delete process.env[key]
}
}
function useOpenAIProvider(): void {
clearProviderEnv()
process.env.CLAUDE_CODE_USE_OPENAI = '1'
process.env.OPENAI_BASE_URL = 'https://api.openai.com/v1'
process.env.OPENAI_MODEL = 'gpt-5.5'
}
function useOpenAIProviderWithDisabledFoundryFlag(flagValue: string): void {
useOpenAIProvider()
process.env.CLAUDE_CODE_USE_FOUNDRY = flagValue
}
function useOpenAIProviderWithWhitespaceBaseFallback(): void {
clearProviderEnv()
process.env.CLAUDE_CODE_USE_OPENAI = '1'
process.env.OPENAI_BASE_URL = ' '
process.env.OPENAI_API_BASE = 'https://api.deepseek.com/v1'
process.env.OPENAI_MODEL = 'deepseek-chat'
}
function useCodexAliasWithLiteralUndefinedBaseUrl(): void {
clearProviderEnv()
process.env.CLAUDE_CODE_USE_OPENAI = '1'
process.env.OPENAI_BASE_URL = 'undefined'
process.env.OPENAI_MODEL = 'codexplan'
}
function useOpenRouterProvider(): void {
clearProviderEnv()
process.env.CLAUDE_CODE_USE_OPENAI = '1'
process.env.OPENAI_BASE_URL = 'https://openrouter.ai/api/v1'
process.env.OPENAI_MODEL = 'gpt-4o'
}
function useUnsupportedGithubProvider(): void {
clearProviderEnv()
process.env.CLAUDE_CODE_USE_GITHUB = '1'
process.env.OPENAI_MODEL = 'gpt-4o'
}
function useFoundryProvider(): void {
clearProviderEnv()
process.env.CLAUDE_CODE_USE_FOUNDRY = '1'
}
function systemBlock(
text: string,
cacheControl?: Record<string, unknown>,
): TextBlockParam[] {
return [
{
type: 'text',
text,
...(cacheControl ? { cache_control: cacheControl } : {}),
},
] as TextBlockParam[]
}
function tool(
name: string,
schema: Record<string, unknown> = {
type: 'object',
properties: {},
},
): BetaToolUnion {
return {
type: 'custom',
name,
description: `${name} test tool`,
input_schema: schema,
} as unknown as BetaToolUnion
}
function assistantMessages(gapMs: number): Message[] {
return [
{
type: 'assistant',
timestamp: new Date(Date.now() - gapMs).toISOString(),
uuid: '00000000-0000-4000-8000-000000000001',
message: {
role: 'assistant',
content: [],
id: 'msg_1',
model: 'claude-sonnet-4',
usage: {
input_tokens: 0,
output_tokens: 0,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
},
},
},
] as unknown as Message[]
}
function snapshot(
overrides: Partial<Parameters<PromptCacheBreakModule['recordPromptState']>[0]> = {},
): Parameters<PromptCacheBreakModule['recordPromptState']>[0] {
return {
system: systemBlock('base system prompt'),
toolSchemas: [tool('Read')],
querySource: 'repl_main_thread' as QuerySource,
model: 'claude-sonnet-4',
...overrides,
}
}
async function triggerCacheDrop({
first = snapshot(),
second = snapshot(),
secondMessages = assistantMessages(60_000),
}: {
first?: Parameters<PromptCacheBreakModule['recordPromptState']>[0]
second?: Parameters<PromptCacheBreakModule['recordPromptState']>[0]
secondMessages?: Message[]
} = {}): Promise<EventCall> {
const mod = await loadDetector()
mod.recordPromptState(first)
await mod.checkResponseForCacheBreak(
first.querySource,
10_000,
0,
assistantMessages(60_000),
first.agentId,
'req-prev',
)
mod.recordPromptState(second)
await mod.checkResponseForCacheBreak(
second.querySource,
1_000,
0,
secondMessages,
second.agentId,
'req-break',
)
const event = events.findLast(e => e.name === 'tengu_prompt_cache_break')
expect(event).toBeDefined()
return event!
}
beforeEach(async () => {
await acquireSharedMutationLock('promptCacheBreakDetection.test.ts')
events.length = 0
debugCalls.length = 0
logEventMock.mockClear()
logForDebuggingMock.mockClear()
clearProviderEnv()
const mod = await loadDetector()
mod.resetPromptCacheBreakDetection()
})
afterEach(() => {
try {
detector?.resetPromptCacheBreakDetection()
restoreProviderEnv()
} finally {
releaseSharedMutationLock()
}
})
describe('prompt cache break taxonomy', () => {
test('tool addition/removal classifies as expected tool schema change', async () => {
const event = await triggerCacheDrop({
first: snapshot({ toolSchemas: [tool('Read'), tool('Edit')] }),
second: snapshot({ toolSchemas: [tool('Edit'), tool('Bash')] }),
})
expect(event.metadata).toMatchObject({
classification: 'expected_tool_schema_change',
addedToolCount: 1,
removedToolCount: 1,
addedTools: 'Bash',
removedTools: 'Read',
})
})
test('changed MCP tool schema names are sanitized in analytics', async () => {
const event = await triggerCacheDrop({
first: snapshot({
toolSchemas: [
tool('mcp__repo_server__search', {
type: 'object',
properties: { query: { type: 'string' } },
}),
],
}),
second: snapshot({
toolSchemas: [
tool('mcp__repo_server__search', {
type: 'object',
properties: { query: { type: 'string' }, limit: { type: 'number' } },
}),
],
}),
})
expect(event.metadata).toMatchObject({
classification: 'expected_tool_schema_change',
changedToolSchemas: 'mcp',
})
})
test('system prompt hash change classifies as expected local prompt change', async () => {
const event = await triggerCacheDrop({
first: snapshot({ system: systemBlock('base system prompt') }),
second: snapshot({ system: systemBlock('updated system prompt') }),
})
expect(event.metadata).toMatchObject({
classification: 'expected_local_prompt_change',
severity: 'info',
})
})
test('cache-control-only change classifies separately', async () => {
const event = await triggerCacheDrop({
first: snapshot({
system: systemBlock('base system prompt', { type: 'ephemeral' }),
}),
second: snapshot({
system: systemBlock('base system prompt', {
type: 'ephemeral',
ttl: '1h',
}),
}),
})
expect(event.metadata).toMatchObject({
classification: 'expected_cache_control_change',
cacheControlChanged: true,
systemPromptChanged: false,
})
})
test('global cache strategy change classifies with cache-control changes', async () => {
const event = await triggerCacheDrop({
first: snapshot({ globalCacheStrategy: 'tool_based' }),
second: snapshot({ globalCacheStrategy: 'system_prompt' }),
})
expect(event.metadata).toMatchObject({
classification: 'expected_cache_control_change',
cacheControlChanged: false,
globalCacheStrategyChanged: true,
prevGlobalCacheStrategy: 'tool_based',
newGlobalCacheStrategy: 'system_prompt',
})
})
test('model, beta, effort, and extra-body changes classify as expected model or beta change', async () => {
const event = await triggerCacheDrop({
first: snapshot({
model: 'claude-sonnet-4',
betas: ['beta-a'],
effortValue: 'medium',
extraBodyParams: { metadata: { source: 'first' } },
}),
second: snapshot({
model: 'claude-opus-4',
betas: ['beta-b'],
effortValue: 'high',
extraBodyParams: { metadata: { source: 'second' } },
}),
})
expect(event.metadata).toMatchObject({
classification: 'expected_model_or_beta_change',
modelChanged: true,
betasChanged: true,
effortChanged: true,
extraBodyChanged: true,
addedBetas: 'beta-b',
removedBetas: 'beta-a',
})
})
test('same prompt/schema under TTL on advisory provider classifies as provider cache instability with info severity', async () => {
useOpenAIProvider()
const event = await triggerCacheDrop()
const debug = debugCalls.findLast(call =>
call.message.startsWith('[PROMPT CACHE BREAK]'),
)
expect(event.metadata).toMatchObject({
classification: 'provider_cache_instability',
cacheMetricsReliability: 'advisory',
severity: 'info',
prevCacheReadTokens: 10_000,
cacheReadTokens: 1_000,
tokenDrop: 9_000,
querySource: 'repl_main_thread',
model: 'claude-sonnet-4',
providerRoute: 'openai',
requestId: 'req-break',
})
expect(debug?.options).toEqual({ level: 'info' })
expect(debug?.message).not.toMatch(/app crash|crash|local mutation/i)
})
for (const falseyFoundryFlag of ['off', 'no'] as const) {
test(`stale Foundry=${falseyFoundryFlag} env flag does not override OpenAI cache metadata`, async () => {
useOpenAIProviderWithDisabledFoundryFlag(falseyFoundryFlag)
const event = await triggerCacheDrop()
const debug = debugCalls.findLast(call =>
call.message.startsWith('[PROMPT CACHE BREAK]'),
)
expect(event.metadata).toMatchObject({
classification: 'provider_cache_instability',
cacheMetricsReliability: 'advisory',
cacheProvider: 'openai',
providerRoute: 'openai',
severity: 'info',
})
expect(debug?.options).toEqual({ level: 'info' })
})
}
test('blank OpenAI base URL falls back to legacy API base for cache provider metadata', async () => {
useOpenAIProviderWithWhitespaceBaseFallback()
const event = await triggerCacheDrop({
first: snapshot({ model: 'deepseek-chat' }),
second: snapshot({ model: 'deepseek-chat' }),
})
expect(event.metadata).toMatchObject({
classification: 'provider_cache_instability',
cacheMetricsReliability: 'advisory',
cacheProvider: 'deepseek',
severity: 'info',
})
})
test('literal undefined OpenAI base URL still reports Codex alias metadata', async () => {
useCodexAliasWithLiteralUndefinedBaseUrl()
const event = await triggerCacheDrop({
first: snapshot({ model: 'codexplan' }),
second: snapshot({ model: 'codexplan' }),
})
expect(event.metadata).toMatchObject({
classification: 'provider_cache_instability',
cacheMetricsReliability: 'advisory',
cacheProvider: 'codex',
providerRoute: 'codex',
severity: 'info',
})
})
test('descriptor OpenAI-compatible route IDs are normalized before logging', async () => {
useOpenRouterProvider()
const event = await triggerCacheDrop({
first: snapshot({ model: 'gpt-4o' }),
second: snapshot({ model: 'gpt-4o' }),
})
const debug = debugCalls.findLast(call =>
call.message.startsWith('[PROMPT CACHE BREAK]'),
)
expect(event.metadata).toMatchObject({
classification: 'provider_cache_instability',
cacheMetricsReliability: 'advisory',
cacheProvider: 'openai',
providerRoute: 'openai-compatible',
severity: 'info',
})
expect(debug?.message).toContain('route=openai-compatible')
expect(debug?.message).not.toContain('openrouter')
})
test('TTL-window cache drops classify as possible TTL expiry', async () => {
const event = await triggerCacheDrop({
secondMessages: assistantMessages(61 * 60 * 1000),
})
expect(event.metadata).toMatchObject({
classification: 'possible_ttl_expiry',
lastAssistantMsgOver5minAgo: true,
lastAssistantMsgOver1hAgo: true,
})
})
test('unknown or incomplete state classifies as unknown local mutation, not provider-side instability', async () => {
const event = await triggerCacheDrop({
secondMessages: [],
})
expect(event.metadata).toMatchObject({
classification: 'unknown_local_mutation',
timeSinceLastAssistantMsg: -1,
})
})
test('unsupported provider metrics classify as metrics unavailable', async () => {
useUnsupportedGithubProvider()
const event = await triggerCacheDrop({
first: snapshot({ model: 'gpt-4o' }),
second: snapshot({ model: 'gpt-4o' }),
})
expect(event.metadata).toMatchObject({
classification: 'metrics_unavailable',
cacheMetricsReliability: 'unsupported',
providerRoute: 'github',
})
})
test('unsupported provider metrics do not hide known local prompt changes', async () => {
useUnsupportedGithubProvider()
const event = await triggerCacheDrop({
first: snapshot({
model: 'gpt-4o',
system: systemBlock('base system prompt'),
}),
second: snapshot({
model: 'gpt-4o',
system: systemBlock('changed system prompt'),
}),
})
expect(event.metadata).toMatchObject({
classification: 'expected_local_prompt_change',
cacheMetricsReliability: 'unsupported',
systemPromptChanged: true,
})
})
test('legacy Foundry provider does not get mislabeled as anthropic route', async () => {
useFoundryProvider()
const event = await triggerCacheDrop()
const debug = debugCalls.findLast(call =>
call.message.startsWith('[PROMPT CACHE BREAK]'),
)
expect(event.metadata).toMatchObject({
classification: 'provider_cache_instability',
cacheMetricsReliability: 'reliable',
cacheProvider: 'anthropic',
providerRoute: 'foundry',
severity: 'warning',
})
expect(debug?.options).toEqual({ level: 'warn' })
})
test('cache deletion expected drops remain suppressed', async () => {
const mod = await loadDetector()
const first = snapshot()
mod.recordPromptState(first)
await mod.checkResponseForCacheBreak(
first.querySource,
10_000,
0,
assistantMessages(60_000),
first.agentId,
'req-prev',
)
mod.recordPromptState(snapshot())
mod.notifyCacheDeletion(first.querySource, first.agentId)
await mod.checkResponseForCacheBreak(
first.querySource,
1_000,
0,
assistantMessages(60_000),
first.agentId,
'req-drop',
)
expect(events.filter(e => e.name === 'tengu_prompt_cache_break')).toEqual([])
expect(
debugCalls.some(call => call.message.includes('cache deletion applied')),
).toBe(true)
})
})
+308 -8
View File
@@ -5,16 +5,25 @@ import { mkdir, writeFile } from 'fs/promises'
import { join } from 'path'
import type { AgentId } from 'src/types/ids.js'
import type { Message } from 'src/types/message.js'
import { logForDebugging } from 'src/utils/debug.js'
import { type DebugLogLevel, logForDebugging } from 'src/utils/debug.js'
import { djb2Hash } from 'src/utils/hash.js'
import { logError } from 'src/utils/log.js'
import type { APIProvider } from 'src/utils/model/providers.js'
import { getClaudeTempDir } from 'src/utils/permissions/filesystem.js'
import { jsonStringify } from 'src/utils/slowOperations.js'
import type { QuerySource } from '../../constants/querySource.js'
import {
getTransportKindForRoute,
resolveActiveRouteIdFromEnv,
} from '../../integrations/routeMetadata.js'
import {
type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
logEvent,
} from '../analytics/index.js'
import {
getCacheMetricsReliability,
resolveCacheProvider,
type CacheMetricsReliability,
} from './cacheMetrics.js'
function getCacheBreakDiffPath(): string {
const chars = 'abcdefghijklmnopqrstuvwxyz0123456789'
@@ -98,6 +107,24 @@ type PendingChanges = {
buildPrevDiffableContent: () => string
}
export type PromptCacheBreakKind =
| 'expected_local_prompt_change'
| 'expected_tool_schema_change'
| 'expected_cache_control_change'
| 'expected_model_or_beta_change'
| 'provider_cache_instability'
| 'possible_ttl_expiry'
| 'unknown_local_mutation'
| 'metrics_unavailable'
type PromptCacheBreakSeverity = 'debug' | 'info' | 'warning'
type PromptCacheBreakClassification = {
kind: PromptCacheBreakKind
severity: PromptCacheBreakSeverity
debugLevel: DebugLogLevel
}
const previousStateBySource = new Map<string, PreviousState>()
// Cap the number of tracked sources to prevent unbounded memory growth.
@@ -168,7 +195,7 @@ function stripCacheControl(
}
function computeHash(data: unknown): number {
const str = jsonStringify(data)
const str = stringifyCacheBreakData(data)
if (typeof Bun !== 'undefined') {
const hash = Bun.hash(str)
// Bun.hash can return bigint for large inputs; convert to number safely
@@ -178,6 +205,40 @@ function computeHash(data: unknown): number {
return djb2Hash(str)
}
function stringifyCacheBreakData(data: unknown): string {
return JSON.stringify(data) ?? ''
}
function isCacheBreakEnvTruthy(value: string | undefined): boolean {
const normalized = value?.trim().toLowerCase()
return (
normalized === '1' ||
normalized === 'true' ||
normalized === 'yes' ||
normalized === 'on'
)
}
function getNonEmptyEnvValue(value: string | undefined): string | undefined {
const trimmed = value?.trim()
return trimmed && trimmed !== 'undefined' ? trimmed : undefined
}
const SAFE_PROMPT_CACHE_PROVIDER_ROUTES = new Set<string>([
'anthropic',
'openai',
'custom',
'gemini',
'mistral',
'github',
'bedrock',
'vertex',
'nvidia-nim',
'minimax',
'xiaomi-mimo',
'xai',
])
/** MCP tool names are user-controlled (server config) and may leak filepaths.
* Collapse them to 'mcp'; built-in names are a fixed vocabulary. */
function sanitizeToolName(name: string): string {
@@ -213,7 +274,8 @@ function buildDiffableContent(
.map(t => {
if (!('name' in t)) return 'unknown'
const desc = 'description' in t ? t.description : ''
const schema = 'input_schema' in t ? jsonStringify(t.input_schema) : ''
const schema =
'input_schema' in t ? stringifyCacheBreakData(t.input_schema) : ''
return `${t.name}\n description: ${desc}\n input_schema: ${schema}`
})
.sort()
@@ -221,6 +283,211 @@ function buildDiffableContent(
return `Model: ${model}\n\n=== System Prompt ===\n\n${systemText}\n\n=== Tools (${tools.length}) ===\n\n${toolDetails}\n`
}
function classifyPromptCacheBreak({
changes,
cacheMetricsReliability,
lastAssistantMsgOver5minAgo,
timeSinceLastAssistantMsg,
}: {
changes: PendingChanges | null
cacheMetricsReliability: CacheMetricsReliability
lastAssistantMsgOver5minAgo: boolean
timeSinceLastAssistantMsg: number | null
}): PromptCacheBreakClassification {
if (changes) {
if (changes.systemPromptChanged) {
return {
kind: 'expected_local_prompt_change',
severity: 'info',
debugLevel: 'info',
}
}
if (changes.toolSchemasChanged) {
return {
kind: 'expected_tool_schema_change',
severity: 'info',
debugLevel: 'info',
}
}
if (
(changes.cacheControlChanged || changes.globalCacheStrategyChanged) &&
!changes.systemPromptChanged
) {
return {
kind: 'expected_cache_control_change',
severity: 'info',
debugLevel: 'info',
}
}
if (
changes.modelChanged ||
changes.fastModeChanged ||
changes.betasChanged ||
changes.autoModeChanged ||
changes.overageChanged ||
changes.cachedMCChanged ||
changes.effortChanged ||
changes.extraBodyChanged
) {
return {
kind: 'expected_model_or_beta_change',
severity: 'info',
debugLevel: 'info',
}
}
}
if (cacheMetricsReliability === 'unsupported') {
return {
kind: 'metrics_unavailable',
severity: 'info',
debugLevel: 'info',
}
}
if (lastAssistantMsgOver5minAgo) {
return {
kind: 'possible_ttl_expiry',
severity: 'info',
debugLevel: 'info',
}
}
if (timeSinceLastAssistantMsg !== null) {
const advisory = cacheMetricsReliability === 'advisory'
return {
kind: 'provider_cache_instability',
severity: advisory ? 'info' : 'warning',
debugLevel: advisory ? 'info' : 'warn',
}
}
return {
kind: 'unknown_local_mutation',
severity: 'warning',
debugLevel: 'warn',
}
}
function getPromptCacheBreakProviderMetadata(model: string): {
cacheProvider: string
cacheMetricsReliability: CacheMetricsReliability
providerRoute: string
} {
const activeRouteId = resolveActiveRouteIdFromEnv(process.env)
const apiProvider = resolvePromptCacheBreakAPIProvider(
process.env,
activeRouteId,
model,
)
const cacheProvider = resolveCacheProvider(apiProvider, {
githubNativeAnthropic: isGithubNativeAnthropicModeForCacheBreak(
process.env,
model,
),
openAiBaseUrl:
getNonEmptyEnvValue(process.env.OPENAI_BASE_URL) ??
getNonEmptyEnvValue(process.env.OPENAI_API_BASE),
})
return {
cacheProvider,
cacheMetricsReliability: getCacheMetricsReliability(cacheProvider),
providerRoute: getPromptCacheBreakProviderRoute(
activeRouteId,
apiProvider,
),
}
}
function getPromptCacheBreakProviderRoute(
activeRouteId: string | null,
apiProvider: APIProvider,
): string {
if (apiProvider === 'codex') {
return 'codex'
}
if (activeRouteId === 'anthropic' && apiProvider !== 'firstParty') {
return apiProvider
}
if (!activeRouteId) {
return apiProvider
}
if (SAFE_PROMPT_CACHE_PROVIDER_ROUTES.has(activeRouteId)) {
return activeRouteId
}
return getTransportKindForRoute(activeRouteId) ?? apiProvider
}
function resolvePromptCacheBreakAPIProvider(
env: NodeJS.ProcessEnv,
activeRouteId: string | null,
model: string,
): APIProvider {
if (isCacheBreakEnvTruthy(env.CLAUDE_CODE_USE_FOUNDRY)) {
return 'foundry'
}
switch (activeRouteId) {
case 'gemini':
case 'mistral':
case 'github':
case 'bedrock':
case 'vertex':
case 'nvidia-nim':
case 'minimax':
case 'xiaomi-mimo':
case 'xai':
return activeRouteId
case 'openai':
case 'custom':
if (isCacheBreakEnvTruthy(env.NVIDIA_NIM)) {
return 'nvidia-nim'
}
return isCodexCacheBreakRoute(env, model) ? 'codex' : 'openai'
case 'anthropic':
case null:
if (isCacheBreakEnvTruthy(env.NVIDIA_NIM)) {
return 'nvidia-nim'
}
return 'firstParty'
default:
if (
['local', 'openai-compatible'].includes(
getTransportKindForRoute(activeRouteId) ?? '',
)
) {
return 'openai'
}
return 'firstParty'
}
}
function isGithubNativeAnthropicModeForCacheBreak(
env: NodeJS.ProcessEnv,
model: string,
): boolean {
if (!isCacheBreakEnvTruthy(env.CLAUDE_CODE_USE_GITHUB)) return false
const resolvedModel = model.trim() || env.OPENAI_MODEL?.trim() || ''
return resolvedModel.toLowerCase().includes('claude-')
}
function isCodexCacheBreakRoute(
env: NodeJS.ProcessEnv,
model: string,
): boolean {
const baseUrl =
getNonEmptyEnvValue(env.OPENAI_BASE_URL) ??
getNonEmptyEnvValue(env.OPENAI_API_BASE)
if (baseUrl?.toLowerCase().includes('/backend-api/codex')) {
return true
}
if (baseUrl?.trim()) {
return false
}
const modelName = (env.OPENAI_MODEL?.trim() || model.trim()).toLowerCase()
return modelName.includes('codex')
}
/** Extended tracking snapshot — everything that could affect the server-side
* cache key that we can observe from the client. All fields are optional so
* the call site can add incrementally; undefined fields compare as stable. */
@@ -491,6 +758,12 @@ export async function checkResponseForCacheBreak(
return
}
const {
cacheProvider,
cacheMetricsReliability,
providerRoute,
} = getPromptCacheBreakProviderMetadata(state.model)
// Build explanation from pending changes (if any)
const parts: string[] = []
if (changes) {
@@ -570,6 +843,13 @@ export async function checkResponseForCacheBreak(
timeSinceLastAssistantMsg !== null &&
timeSinceLastAssistantMsg > CACHE_TTL_1HOUR_MS
const classification = classifyPromptCacheBreak({
changes,
cacheMetricsReliability,
lastAssistantMsgOver5minAgo,
timeSinceLastAssistantMsg,
})
// Post PR #19823 BQ analysis (bq-queries/prompt-caching/cache_break_pr19823_analysis.sql):
// when all client-side flags are false and the gap is under TTL, ~90% of breaks
// are server-side routing/eviction or billed/inference disagreement. Label
@@ -577,17 +857,36 @@ export async function checkResponseForCacheBreak(
let reason: string
if (parts.length > 0) {
reason = parts.join(', ')
} else if (classification.kind === 'metrics_unavailable') {
reason = `cache metrics unavailable or unsupported for provider (${cacheProvider})`
} else if (lastAssistantMsgOver1hAgo) {
reason = 'possible 1h TTL expiry (prompt unchanged)'
} else if (lastAssistantMsgOver5minAgo) {
reason = 'possible 5min TTL expiry (prompt unchanged)'
} else if (timeSinceLastAssistantMsg !== null) {
reason = 'likely server-side (prompt unchanged, <5min gap)'
reason =
cacheMetricsReliability === 'advisory'
? 'provider-side cache metric instability (prompt unchanged, <5min gap, advisory metrics)'
: 'provider-side cache instability (prompt unchanged, <5min gap)'
} else {
reason = 'unknown cause'
reason = 'unknown local mutation or incomplete prompt state'
}
logEvent('tengu_prompt_cache_break', {
classification:
classification.kind as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
severity:
classification.severity as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
cacheMetricsReliability:
cacheMetricsReliability as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
cacheProvider:
cacheProvider as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
providerRoute:
providerRoute as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
querySource:
querySource as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
model:
state.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
systemPromptChanged: changes?.systemPromptChanged ?? false,
toolSchemasChanged: changes?.toolSchemasChanged ?? false,
modelChanged: changes?.modelChanged ?? false,
@@ -636,6 +935,7 @@ export async function checkResponseForCacheBreak(
prevCacheReadTokens: prevCacheRead,
cacheReadTokens,
cacheCreationTokens,
tokenDrop,
timeSinceLastAssistantMsg: timeSinceLastAssistantMsg ?? -1,
lastAssistantMsgOver5minAgo,
lastAssistantMsgOver1hAgo,
@@ -655,9 +955,9 @@ export async function checkResponseForCacheBreak(
}
const diffSuffix = diffPath ? `, diff: ${diffPath}` : ''
const summary = `[PROMPT CACHE BREAK] ${reason} [source=${querySource}, call #${state.callCount}, cache read: ${prevCacheRead}${cacheReadTokens}, creation: ${cacheCreationTokens}${diffSuffix}]`
const summary = `[PROMPT CACHE BREAK] ${reason} [classification=${classification.kind}, severity=${classification.severity}, reliability=${cacheMetricsReliability}, provider=${cacheProvider}, route=${providerRoute || 'unknown'}, source=${querySource}, call #${state.callCount}, cache read: ${prevCacheRead}${cacheReadTokens}, drop: ${tokenDrop}, creation: ${cacheCreationTokens}${diffSuffix}]`
logForDebugging(summary, { level: 'warn' })
logForDebugging(summary, { level: classification.debugLevel })
state.pendingChanges = null
} catch (e: unknown) {