mirror of
https://github.com/Gitlawb/openclaude.git
synced 2026-11-06 12:50:54 -04:00
fix(query): terminate turns after consecutive doom-loop blocks (#2460)
* fix(query): terminate turns after consecutive doom-loop blocks * fix(query): isolate doom-loop turns and release terminal resources * fix(query): isolate recovered approval doom-loop state * test(query): cover parallel task success after doom termination
This commit is contained in:
@@ -573,6 +573,14 @@ host. Without this variable the behavior is unchanged.
|
||||
| `OPENCLAUDE_DISABLE_CO_AUTHORED_BY` | No | Suppress the default `Co-Authored-By` trailer in generated git commits |
|
||||
| `OPENCLAUDE_LOG_TOKEN_USAGE` | No | When truthy (e.g. `verbose`), emits one JSON line on stderr per API request with input/output/cache tokens and the resolved provider. **User-facing debug output** — complements the REPL display controlled by `/config showCacheStats`. Distinct from `CLAUDE_CODE_ENABLE_TOKEN_USAGE_ATTACHMENT`, which is **model-facing** (injects context usage info into the prompt itself). Both can run together. |
|
||||
|
||||
Repeated identical tool calls are blocked starting with the third call. If a model
|
||||
ignores that feedback, five consecutive doom-loop blocks force-terminate its turn
|
||||
without another model request. A non-blocked call or successful tool result resets
|
||||
the consecutive-block count; each agent and new user turn tracks its own streak.
|
||||
Headless runs (`--bare -p`) print the terminal error and exit with status 1. Change
|
||||
the approach or tool arguments before retrying. This safeguard applies even when
|
||||
the general tool-failure threshold or turn limit is disabled.
|
||||
|
||||
Model env vars are provider-scoped: first-party Anthropic sessions read
|
||||
`ANTHROPIC_MODEL`, OpenAI-compatible sessions read `OPENAI_MODEL`, Gemini reads
|
||||
`GEMINI_MODEL`, Gemini on Vertex reads `GEMINI_VERTEX_MODEL`, and Mistral reads
|
||||
|
||||
@@ -290,6 +290,8 @@ export type ToolUseContext = {
|
||||
) => void
|
||||
setConversationId?: (id: UUID) => void
|
||||
agentId?: AgentId // Only set for subagents; use getSessionId() for session ID. Hooks use this to distinguish subagent calls.
|
||||
/** Private query-turn identity, preserved by tool-context copies. */
|
||||
doomLoopKey?: object
|
||||
/** Last submitted transcript for this producer; undefined for other IDs. */
|
||||
getAgentTranscriptPath?: (agentId: AgentId) => string | undefined
|
||||
agentType?: string // Subagent type name. For the main thread's --agent type, hooks fall back to getMainThreadAgentType().
|
||||
|
||||
+41
-6
@@ -674,7 +674,19 @@ export async function* query(
|
||||
) {
|
||||
params.toolUseContext.queryModelSelection = { agentId: params.toolUseContext.agentId }
|
||||
}
|
||||
const terminal = yield* queryLoop(params, consumedCommandUuids)
|
||||
// SDK sessions can query concurrently without an agentId. A fresh private
|
||||
// key also separates nested/subsequent queries that inherit a tool context.
|
||||
const doomLoopKey = {}
|
||||
const { resetDoomLoop } = await import('./utils/doomLoop.js')
|
||||
let terminal: Terminal
|
||||
try {
|
||||
terminal = yield* queryLoop({
|
||||
...params,
|
||||
toolUseContext: { ...params.toolUseContext, doomLoopKey },
|
||||
}, consumedCommandUuids)
|
||||
} finally {
|
||||
resetDoomLoop(doomLoopKey)
|
||||
}
|
||||
// Only reached if queryLoop returned normally. Skipped on throw (error
|
||||
// propagates through yield*) and on .return() (Return completion closes
|
||||
// both generators). This gives the same asymmetric started-without-completed
|
||||
@@ -696,11 +708,7 @@ async function* queryLoop(
|
||||
| ToolUseSummaryMessage,
|
||||
Terminal
|
||||
> {
|
||||
// Reset this agent's doom loop detection at the start of each query turn.
|
||||
// Keyed by agentId so a subagent starting mid-turn doesn't wipe the main
|
||||
// thread's counter (or a sibling agent's).
|
||||
const { resetDoomLoop } = await import('./utils/doomLoop.js')
|
||||
resetDoomLoop(params.toolUseContext.agentId)
|
||||
const { getDoomLoopState } = await import('./utils/doomLoop.js')
|
||||
|
||||
// Start a new turn for multi-turn context tracking
|
||||
if (
|
||||
@@ -1620,6 +1628,10 @@ async function* queryLoop(
|
||||
queryTracking,
|
||||
queryLifecycle: toolUseContext.queryLifecycle,
|
||||
onProviderRequestStart: () => {
|
||||
if (getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage) {
|
||||
providerDispatchRejected = true
|
||||
return false
|
||||
}
|
||||
if (toolUseContext.abortController.signal.aborted) {
|
||||
providerDispatchRejected = true
|
||||
return false
|
||||
@@ -1862,6 +1874,7 @@ async function* queryLoop(
|
||||
}
|
||||
}
|
||||
if (providerDispatchRejected) {
|
||||
if (getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage) break
|
||||
if (toolUseContext.abortController.signal.aborted) {
|
||||
return yield* emitAbortedStreaming(
|
||||
toolUseContext.abortController.signal,
|
||||
@@ -1903,6 +1916,9 @@ async function* queryLoop(
|
||||
}
|
||||
}
|
||||
} catch (innerError) {
|
||||
// Streaming tools can trip the breaker before the model stream ends.
|
||||
// Preserve their results and terminate instead of dispatching a fallback.
|
||||
if (getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage) break
|
||||
if (innerError instanceof FallbackTriggeredError && fallbackModel) {
|
||||
// Fallback was triggered - switch model and retry
|
||||
currentModel = fallbackModel
|
||||
@@ -2063,6 +2079,18 @@ async function* queryLoop(
|
||||
return { reason: 'model_error', error }
|
||||
}
|
||||
|
||||
const streamingDoomTerminal = getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage
|
||||
if (streamingDoomTerminal) {
|
||||
if (streamingToolExecutor) {
|
||||
for await (const update of streamingToolExecutor.getRemainingResults()) {
|
||||
if (update.message) yield update.message
|
||||
}
|
||||
}
|
||||
await cleanupComputerUseAtTerminal(toolUseContext)
|
||||
yield createAssistantAPIErrorMessage({ content: streamingDoomTerminal })
|
||||
return { reason: 'tool_failure_loop' }
|
||||
}
|
||||
|
||||
// Execute post-sampling hooks after model response is complete
|
||||
if (assistantMessages.length > 0) {
|
||||
void executePostSamplingHooks(
|
||||
@@ -2893,6 +2921,13 @@ async function* queryLoop(
|
||||
)
|
||||
}
|
||||
|
||||
const doomLoopTerminal = getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage
|
||||
if (doomLoopTerminal) {
|
||||
await cleanupComputerUseAtTerminal(toolUseContext)
|
||||
yield createAssistantAPIErrorMessage({ content: doomLoopTerminal })
|
||||
return { reason: 'tool_failure_loop' }
|
||||
}
|
||||
|
||||
// If a hook indicated to prevent continuation, stop here
|
||||
if (shouldPreventContinuation) {
|
||||
return { reason: 'hook_stopped' }
|
||||
|
||||
@@ -1,10 +1,22 @@
|
||||
import type { Message } from '../types/message.js'
|
||||
import { expect, test } from 'bun:test'
|
||||
import { expect, spyOn, test } from 'bun:test'
|
||||
import * as queryConfig from './config.js'
|
||||
import { FallbackTriggeredError } from '../services/api/withRetry.js'
|
||||
import { acquireSharedMutationLock, releaseSharedMutationLock } from '../test/sharedMutationLock.js'
|
||||
import { z } from 'zod/v4'
|
||||
import { createToolFixture } from '../test/toolFixtures.js'
|
||||
import { createTaskStateBase } from '../Task.js'
|
||||
import type { LocalShellTaskState } from '../tasks/LocalShellTask/guards.js'
|
||||
import { TaskOutputTool } from '../tools/TaskOutputTool/TaskOutputTool.js'
|
||||
|
||||
import type { ToolUseBlock } from '@anthropic-ai/sdk/resources/index.mjs'
|
||||
import { query, type QueryParams } from '../query.js'
|
||||
import type { QueryDeps } from './deps.js'
|
||||
import { getMissingToolResultAbortMessage } from '../utils/abortReasons.js'
|
||||
import { resetDoomLoop } from '../utils/doomLoop.js'
|
||||
import { handleOrphanedPermission } from '../utils/queryHelpers.js'
|
||||
import { withTimeout } from '../utils/sleep.js'
|
||||
import type { ProcessUserInputContext } from '../utils/processUserInput/processUserInput.js'
|
||||
import {
|
||||
createAssistantMessage,
|
||||
createCompactBoundaryMessage,
|
||||
@@ -19,6 +31,301 @@ import {
|
||||
|
||||
const querySourceFile = new URL('../query.ts', import.meta.url)
|
||||
|
||||
test('independent recovered approvals do not inherit another turn’s doom state', async () => {
|
||||
resetDoomLoop()
|
||||
const executedValues: number[] = []
|
||||
const resultIds: string[] = []
|
||||
const errors: string[] = []
|
||||
const tool = createToolFixture(z.object({ value: z.number() }), {
|
||||
name: 'RecoveredApprovalTool',
|
||||
maxResultSizeChars: 1000,
|
||||
async call({ value }) {
|
||||
executedValues.push(value)
|
||||
return { data: 'ok' }
|
||||
},
|
||||
})
|
||||
try {
|
||||
for (let i = 0; i < 8; i++) {
|
||||
const params = makeQueryParams(async function* () {
|
||||
yield createAssistantMessage({ content: 'done' })
|
||||
} as QueryDeps['callModel'], { querySource: 'sdk' })
|
||||
params.toolUseContext.options.tools = [tool]
|
||||
const context: ProcessUserInputContext = {
|
||||
...params.toolUseContext,
|
||||
options: { ...params.toolUseContext.options, ideInstallationStatus: null, theme: 'dark' },
|
||||
setMessages: () => {},
|
||||
onChangeAPIKey: () => {},
|
||||
}
|
||||
const id = `recovered-${i}`
|
||||
const input = { value: i < 7 ? 1 : 2 }
|
||||
for await (const message of handleOrphanedPermission({
|
||||
permissionResult: { behavior: 'allow', toolUseID: id, updatedInput: input },
|
||||
assistantMessage: createAssistantMessage({ content: [toolUse(id, tool.name, input)] }),
|
||||
}, [tool], [], context)) {
|
||||
if (message.type !== 'user' || !Array.isArray(message.message.content)) continue
|
||||
for (const block of message.message.content) {
|
||||
if (
|
||||
typeof block !== 'object' || block === null ||
|
||||
!('type' in block) || block.type !== 'tool_result' ||
|
||||
!('tool_use_id' in block) || typeof block.tool_use_id !== 'string'
|
||||
) continue
|
||||
resultIds.push(block.tool_use_id)
|
||||
if ('is_error' in block && block.is_error) errors.push(JSON.stringify(block))
|
||||
}
|
||||
}
|
||||
// QueryEngine continues with an ordinary query after the recovered tool.
|
||||
for await (const _message of query(params)) { /* drain */ }
|
||||
}
|
||||
expect(executedValues).toEqual([1, 1, 1, 1, 1, 1, 1, 2])
|
||||
expect(resultIds).toEqual(Array.from({ length: 8 }, (_, i) => `recovered-${i}`))
|
||||
expect(errors).toEqual([])
|
||||
} finally {
|
||||
resetDoomLoop()
|
||||
}
|
||||
})
|
||||
|
||||
test('concurrent SDK turns cannot reset or terminate each other', async () => {
|
||||
function session(name: string) {
|
||||
let calls = 0
|
||||
const tool = createToolFixture(z.object({}), {
|
||||
name,
|
||||
async call() { return { data: 'ok' } },
|
||||
})
|
||||
const params = makeQueryParams(async function* () {
|
||||
calls++
|
||||
yield createAssistantMessage({
|
||||
content: calls <= 8 ? [toolUse(`${name}-${calls}`, name)] : 'done',
|
||||
})
|
||||
} as QueryDeps['callModel'], { querySource: 'sdk' })
|
||||
params.toolUseContext.options.tools = [tool]
|
||||
return { turn: query(params), calls: () => calls }
|
||||
}
|
||||
const first = session('FirstSessionTool')
|
||||
const second = session('SecondSessionTool')
|
||||
// Suspend the first turn after four blocks, while its generator still owns
|
||||
// its state; run the second session all the way through terminal cleanup.
|
||||
let next = await first.turn.next()
|
||||
while (!next.done) {
|
||||
if (first.calls() === 6 && next.value.type === 'user') break
|
||||
next = await first.turn.next()
|
||||
}
|
||||
let secondTerminal: Message | undefined
|
||||
for await (const message of second.turn) {
|
||||
if (message.type === 'assistant') secondTerminal = message
|
||||
}
|
||||
let firstTerminal: Message | undefined
|
||||
for await (const message of first.turn) {
|
||||
if (message.type === 'assistant') firstTerminal = message
|
||||
}
|
||||
expect(first.calls()).toBe(7)
|
||||
expect(second.calls()).toBe(7)
|
||||
expect(JSON.stringify(firstTerminal)).toContain('FirstSessionTool was blocked 5 consecutive')
|
||||
expect(JSON.stringify(secondTerminal)).toContain('SecondSessionTool was blocked 5 consecutive')
|
||||
})
|
||||
|
||||
test('streaming doom termination wins over fallback and resets on unknown calls', async () => {
|
||||
await acquireSharedMutationLock('doom-loop-streaming-config')
|
||||
const config = queryConfig.buildQueryConfig()
|
||||
const configSpy = spyOn(queryConfig, 'buildQueryConfig').mockReturnValue({
|
||||
...config,
|
||||
gates: { ...config.gates, streamingToolExecution: true },
|
||||
})
|
||||
try {
|
||||
let modelCalls = 0
|
||||
const tool = createToolFixture(z.object({}), {
|
||||
name: 'StreamingDoomTool',
|
||||
async call() { return { data: 'ok' } },
|
||||
})
|
||||
const params = makeQueryParams(async function* () {
|
||||
modelCalls++
|
||||
yield createAssistantMessage({ content: [
|
||||
toolUse(`stream-${modelCalls}`, modelCalls === 7 ? 'UnknownTool' : tool.name),
|
||||
] })
|
||||
if (modelCalls === 12) throw new FallbackTriggeredError('test-model', 'fallback')
|
||||
if (modelCalls > 12) throw new Error('Unexpected fallback model request')
|
||||
} as QueryDeps['callModel'], { fallbackModel: 'fallback' })
|
||||
params.toolUseContext.options.tools = [tool]
|
||||
let terminal: Message | undefined
|
||||
const resultIds: string[] = []
|
||||
for await (const message of query(params)) {
|
||||
if (message.type === 'assistant') terminal = message
|
||||
if (message.type === 'user' && Array.isArray(message.message.content)) {
|
||||
for (const block of message.message.content) {
|
||||
if (block.type === 'tool_result') resultIds.push(block.tool_use_id)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Two allowed, four blocked, unknown (resets), then five blocked calls.
|
||||
expect(modelCalls).toBe(12)
|
||||
expect(resultIds).toEqual(Array.from({ length: 12 }, (_, i) => `stream-${i + 1}`))
|
||||
expect(terminal).toMatchObject({ type: 'assistant', isApiErrorMessage: true })
|
||||
expect(JSON.stringify(terminal)).toContain('force-terminated')
|
||||
} finally {
|
||||
configSpy.mockRestore()
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
for (const streaming of [false, true]) {
|
||||
test(`post-latch task success cannot resume execution (streaming=${streaming})`, async () => {
|
||||
await acquireSharedMutationLock('doom-loop-post-latch')
|
||||
const config = queryConfig.buildQueryConfig()
|
||||
const configSpy = spyOn(queryConfig, 'buildQueryConfig').mockReturnValue({
|
||||
...config, gates: { ...config.gates, streamingToolExecution: streaming },
|
||||
})
|
||||
const task: LocalShellTaskState = {
|
||||
...createTaskStateBase('bdoomlate', 'local_bash', 'pending task'),
|
||||
type: 'local_bash', status: 'running', command: '', shellCommand: null,
|
||||
completionStatusSentInAttachment: false, lastReportedTotalLines: 0, isBackgrounded: true,
|
||||
}
|
||||
let started = 0
|
||||
let markStarted!: () => void
|
||||
const bothStarted = new Promise<void>(resolve => { markStarted = resolve })
|
||||
const pending: Promise<unknown>[] = []
|
||||
const taskTool = {
|
||||
...TaskOutputTool,
|
||||
call(...args: Parameters<typeof TaskOutputTool.call>) {
|
||||
if (++started === 2) markStarted()
|
||||
const result = TaskOutputTool.call(...args)
|
||||
pending.push(result)
|
||||
return result
|
||||
},
|
||||
}
|
||||
let extraExecutions = 0
|
||||
const afterLatchTool = createToolFixture(z.object({}), {
|
||||
name: 'AfterLatchTool',
|
||||
async call() { extraExecutions++; return { data: 'unexpected' } },
|
||||
})
|
||||
let modelCalls = 0
|
||||
const ids = Array.from({ length: 7 }, (_, i) => `parallel-${i + 1}`)
|
||||
const params = makeQueryParams(async function* () {
|
||||
modelCalls++
|
||||
yield createAssistantMessage({ content: modelCalls === 1
|
||||
? [...ids.map(id => toolUse(id, taskTool.name, { task_id: task.id, block: true, timeout: 30000 })),
|
||||
toolUse('after-latch', afterLatchTool.name)]
|
||||
: 'done' })
|
||||
} as QueryDeps['callModel'])
|
||||
params.toolUseContext.options.tools = [taskTool, afterLatchTool]
|
||||
const getAppState = params.toolUseContext.getAppState
|
||||
params.toolUseContext.getAppState = () => ({ ...getAppState(), tasks: { [task.id]: task } })
|
||||
params.toolUseContext.setAppState = () => {}
|
||||
const results: { id: string; error: boolean; content: string }[] = []
|
||||
let terminal: Message | undefined
|
||||
try {
|
||||
for await (const message of query(params)) {
|
||||
if (message.type === 'assistant') terminal = message
|
||||
if (message.type !== 'user' || !Array.isArray(message.message.content)) continue
|
||||
for (const block of message.message.content) {
|
||||
if (block.type !== 'tool_result') continue
|
||||
results.push({ id: block.tool_use_id, error: Boolean(block.is_error), content: String(block.content) })
|
||||
if (block.tool_use_id === 'parallel-7') {
|
||||
expect(String(block.content)).toContain('Blocked: 7 consecutive calls')
|
||||
await withTimeout(bothStarted, 2000, 'Both permitted task calls must enter their wait')
|
||||
expect(task.status).toBe('running')
|
||||
task.status = 'completed'
|
||||
task.result = { code: 0, interrupted: false }
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(started).toBe(2)
|
||||
expect(results.map(result => result.id).sort()).toEqual([...ids, 'after-latch'].sort())
|
||||
for (const id of ['parallel-1', 'parallel-2']) {
|
||||
expect(results.find(result => result.id === id)).toMatchObject({ error: false, content: expect.stringContaining('<retrieval_status>success') })
|
||||
}
|
||||
expect(results.find(result => result.id === 'after-latch')?.error).toBe(true)
|
||||
expect(extraExecutions).toBe(0)
|
||||
expect(modelCalls).toBe(1)
|
||||
expect(terminal).toMatchObject({ type: 'assistant', isApiErrorMessage: true })
|
||||
expect(JSON.stringify(terminal)).toContain('force-terminated')
|
||||
} finally {
|
||||
task.status = 'completed'
|
||||
params.toolUseContext.abortController.abort()
|
||||
await Promise.allSettled(pending)
|
||||
configSpy.mockRestore()
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
test('doom-loop blocks terminate the query after five consecutive blocks', async () => {
|
||||
let modelCalls = 0
|
||||
let executions = 0
|
||||
const tool = createToolFixture(z.object({ value: z.string() }), {
|
||||
name: 'DoomLoopTestTool',
|
||||
async call() {
|
||||
executions++
|
||||
return { data: 'ok' }
|
||||
},
|
||||
})
|
||||
const params = makeQueryParams(async function* () {
|
||||
modelCalls++
|
||||
// Bound the broken baseline without relying on a timeout to prove failure.
|
||||
yield createAssistantMessage({
|
||||
content: modelCalls <= 8
|
||||
? [toolUse(`repeat-${modelCalls}`, tool.name, { value: 'same' })]
|
||||
: 'model finally stopped',
|
||||
})
|
||||
} as QueryDeps['callModel'])
|
||||
params.toolUseContext.options.tools = [tool]
|
||||
const messages: Message[] = []
|
||||
const turn = query(params)
|
||||
let next = await turn.next()
|
||||
while (!next.done) {
|
||||
if (next.value.type === 'assistant') messages.push(next.value)
|
||||
next = await turn.next()
|
||||
}
|
||||
expect(executions).toBe(2)
|
||||
expect(modelCalls).toBe(7)
|
||||
expect(next.value).toEqual({ reason: 'tool_failure_loop' })
|
||||
const terminal = messages.at(-1)
|
||||
expect(terminal).toMatchObject({ type: 'assistant', isApiErrorMessage: true })
|
||||
if (terminal?.type !== 'assistant') throw new Error('Missing terminal error')
|
||||
expect(terminal.message.content).toEqual([
|
||||
expect.objectContaining({
|
||||
type: 'text',
|
||||
text: expect.stringContaining('force-terminated'),
|
||||
}),
|
||||
])
|
||||
const text = JSON.stringify(terminal.message.content)
|
||||
expect(text).toContain(tool.name)
|
||||
expect(text).toContain('5 consecutive')
|
||||
expect(text).toContain('Blocked: 7 consecutive calls')
|
||||
|
||||
// A subsequent user turn starts clean even after a latched termination.
|
||||
modelCalls = 0
|
||||
for await (const _message of query({ ...params, maxTurns: 1 })) { /* drain */ }
|
||||
expect(executions).toBe(3)
|
||||
expect(modelCalls).toBe(1)
|
||||
})
|
||||
|
||||
test('changed arguments reset doom blocks during a progressing query', async () => {
|
||||
let modelCalls = 0
|
||||
let executions = 0
|
||||
const tool = createToolFixture(z.object({ value: z.number() }), {
|
||||
name: 'DoomLoopProgressTool',
|
||||
async call() {
|
||||
executions++
|
||||
return { data: 'ok' }
|
||||
},
|
||||
})
|
||||
const params = makeQueryParams(async function* () {
|
||||
modelCalls++
|
||||
yield createAssistantMessage({
|
||||
content: modelCalls <= 12
|
||||
? [toolUse(`progress-${modelCalls}`, tool.name, {
|
||||
value: modelCalls <= 6 ? 1 : 2,
|
||||
})]
|
||||
: 'done',
|
||||
})
|
||||
} as QueryDeps['callModel'])
|
||||
params.toolUseContext.options.tools = [tool]
|
||||
for await (const message of query(params)) {
|
||||
if (message.type === 'assistant') expect(message.isApiErrorMessage).not.toBe(true)
|
||||
}
|
||||
expect(modelCalls).toBe(13)
|
||||
expect(executions).toBe(4)
|
||||
})
|
||||
|
||||
function toolUse(
|
||||
id: string,
|
||||
name: string,
|
||||
|
||||
@@ -9,6 +9,7 @@ import { findToolByName, type Tools, type ToolUseContext } from '../../Tool.js'
|
||||
import { BASH_TOOL_NAME } from '../../tools/BashTool/toolName.js'
|
||||
import type { AssistantMessage, Message } from '../../types/message.js'
|
||||
import { createChildAbortController } from '../../utils/abortController.js'
|
||||
import { recordDoomLoopProgress } from '../../utils/doomLoop.js'
|
||||
import {
|
||||
getMissingToolResultAbortMessage,
|
||||
shouldCreateUserInterruptionMessage,
|
||||
@@ -118,6 +119,7 @@ export class StreamingToolExecutor {
|
||||
addTool(block: ToolUseBlock, assistantMessage: AssistantMessage): void {
|
||||
const toolDefinition = findToolByName(this.toolDefinitions, block.name)
|
||||
if (!toolDefinition) {
|
||||
recordDoomLoopProgress(this.toolUseContext.doomLoopKey ?? this.toolUseContext.agentId)
|
||||
this.tools.push({
|
||||
id: block.id,
|
||||
block,
|
||||
|
||||
@@ -16,7 +16,9 @@ import { BASH_TOOL_NAME } from '../../tools/BashTool/toolName.js'
|
||||
import { FILE_EDIT_TOOL_NAME } from '../../tools/FileEditTool/constants.js'
|
||||
import { FILE_WRITE_TOOL_NAME } from '../../tools/FileWriteTool/constants.js'
|
||||
import { NOTEBOOK_EDIT_TOOL_NAME } from '../../tools/NotebookEditTool/constants.js'
|
||||
import { asAgentId } from '../../types/ids.js'
|
||||
import { AbortError } from '../../utils/errors.js'
|
||||
import { getDoomLoopState, resetDoomLoop } from '../../utils/doomLoop.js'
|
||||
import { CANCEL_MESSAGE, createAssistantMessage } from '../../utils/messages.js'
|
||||
import {
|
||||
type QueryActiveOperationSnapshot,
|
||||
@@ -144,6 +146,55 @@ async function collectRunToolUse(
|
||||
return updates
|
||||
}
|
||||
|
||||
test('a late successful tool result resets doom blocks without clearing another agent', async () => {
|
||||
const agentId = asAgentId('doom-loop-late-success')
|
||||
const siblingId = asAgentId('doom-loop-sibling')
|
||||
let finish!: () => void
|
||||
let started!: () => void
|
||||
const ready = new Promise<void>(resolve => { started = resolve })
|
||||
const pending = new Promise<void>(resolve => { finish = resolve })
|
||||
let calls = 0
|
||||
const tool = createToolFixture(z.object({ value: z.string() }), {
|
||||
name: 'DoomLoopLateSuccessTool',
|
||||
async call() {
|
||||
if (++calls === 2) {
|
||||
started()
|
||||
await pending
|
||||
}
|
||||
return { data: 'ok' }
|
||||
},
|
||||
})
|
||||
const context = createLifecycleToolUseContext([tool], new CountingQueryLifecycleTracker())
|
||||
context.agentId = agentId
|
||||
const sibling = { ...context, agentId: siblingId }
|
||||
let inFlight: Promise<MessageUpdateLazy[]> | undefined
|
||||
try {
|
||||
await collectRunToolUse(tool, { value: 'same' }, context)
|
||||
inFlight = collectRunToolUse(tool, { value: 'same' }, context)
|
||||
await ready
|
||||
for (let i = 0; i < 4; i++) await collectRunToolUse(tool, { value: 'same' }, context)
|
||||
for (let i = 0; i < 6; i++) await collectRunToolUse(tool, { value: 'same' }, sibling)
|
||||
expect(getDoomLoopState(agentId).consecutiveBlocks).toBe(4)
|
||||
finish()
|
||||
await inFlight
|
||||
expect(getDoomLoopState(agentId).consecutiveBlocks).toBe(0)
|
||||
expect(getDoomLoopState(siblingId).consecutiveBlocks).toBe(4)
|
||||
for (let i = 0; i < 4; i++) await collectRunToolUse(tool, { value: 'same' }, context)
|
||||
expect(getDoomLoopState(agentId).terminalMessage).toBeNull()
|
||||
await collectRunToolUse(tool, { value: 'same' }, context)
|
||||
expect(getDoomLoopState(agentId).terminalMessage).toContain('5 consecutive')
|
||||
const executedBefore = calls
|
||||
const skipped = await collectRunToolUse(tool, { value: 'different' }, context)
|
||||
expect(calls).toBe(executedBefore)
|
||||
expect(firstToolResultText(skipped[0]?.message)).toContain('force-terminated')
|
||||
} finally {
|
||||
finish()
|
||||
await inFlight
|
||||
resetDoomLoop(agentId)
|
||||
resetDoomLoop(siblingId)
|
||||
}
|
||||
})
|
||||
|
||||
describe('getSchemaValidationErrorOverride', () => {
|
||||
test('returns actionable missing-skill error for SkillTool', () => {
|
||||
expect(getSchemaValidationErrorOverride(SkillTool, {})).toBe(
|
||||
|
||||
@@ -63,7 +63,12 @@ import {
|
||||
shouldCreateUserInterruptionMessage,
|
||||
} from '../../utils/abortReasons.js'
|
||||
import { logForDebugging } from '../../utils/debug.js'
|
||||
import { checkDoomLoop } from '../../utils/doomLoop.js'
|
||||
import {
|
||||
checkDoomLoop,
|
||||
getDoomLoopState,
|
||||
recordDoomLoopBlock,
|
||||
recordDoomLoopProgress,
|
||||
} from '../../utils/doomLoop.js'
|
||||
import {
|
||||
AbortError,
|
||||
errorMessage,
|
||||
@@ -408,6 +413,22 @@ export async function* runToolUse(
|
||||
toolUseContext: ToolUseContext,
|
||||
): AsyncGenerator<MessageUpdateLazy, void> {
|
||||
const toolName = toolUse.name
|
||||
const terminalMessage = getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage
|
||||
if (terminalMessage) {
|
||||
// Preserve tool-use/result pairing without starting more work in this batch.
|
||||
yield {
|
||||
message: createUserMessage({
|
||||
content: [{
|
||||
type: 'tool_result',
|
||||
tool_use_id: toolUse.id,
|
||||
content: terminalMessage,
|
||||
is_error: true,
|
||||
}],
|
||||
sourceToolAssistantUUID: assistantMessage.uuid,
|
||||
}),
|
||||
}
|
||||
return
|
||||
}
|
||||
// First try to find in the available tools (what the model sees)
|
||||
let tool = findToolByName(toolUseContext.options.tools, toolName)
|
||||
|
||||
@@ -434,6 +455,7 @@ export async function* runToolUse(
|
||||
|
||||
// Check if the tool exists
|
||||
if (!tool) {
|
||||
recordDoomLoopProgress(toolUseContext.doomLoopKey ?? toolUseContext.agentId)
|
||||
const sanitizedToolName = sanitizeToolNameForAnalytics(toolName)
|
||||
logForDebugging(`Unknown tool ${toolName}: ${toolUse.id}`)
|
||||
logEvent('tengu_tool_use_error', {
|
||||
@@ -480,9 +502,11 @@ export async function* runToolUse(
|
||||
// Doom loop detection: block after N consecutive identical tool calls.
|
||||
// Keyed by agent so concurrent subagents don't pollute each other's counters.
|
||||
const doomLoop = checkDoomLoop(toolName, toolUse.input, {
|
||||
agentKey: toolUseContext.agentId,
|
||||
agentKey: toolUseContext.doomLoopKey ?? toolUseContext.agentId,
|
||||
})
|
||||
if (doomLoop.blocked) {
|
||||
const error = `Blocked: ${doomLoop.count} consecutive calls to this tool with identical input — you are likely in a loop. Change the input, try a different tool, or ask the user for help. A deliberate repeat is fine once something observable has changed (new output to check, a modified file, an updated instruction).`
|
||||
recordDoomLoopBlock(toolName, error, toolUseContext.doomLoopKey ?? toolUseContext.agentId)
|
||||
logForDebugging(`Doom loop detected for ${toolName} (${doomLoop.count} consecutive identical calls)`)
|
||||
// Observability for tuning: a burst of these against varied tools would
|
||||
// indicate false positives (e.g. legitimate polling), not stuck agents.
|
||||
@@ -497,7 +521,7 @@ export async function* runToolUse(
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
content: `<tool_use_error>Blocked: ${doomLoop.count} consecutive calls to this tool with identical input — you are likely in a loop. Change the input, try a different tool, or ask the user for help. A deliberate repeat is fine once something observable has changed (new output to check, a modified file, an updated instruction).</tool_use_error>`,
|
||||
content: `<tool_use_error>${error}</tool_use_error>`,
|
||||
is_error: true,
|
||||
tool_use_id: toolUse.id,
|
||||
},
|
||||
@@ -509,6 +533,7 @@ export async function* runToolUse(
|
||||
return
|
||||
}
|
||||
|
||||
recordDoomLoopProgress(toolUseContext.doomLoopKey ?? toolUseContext.agentId)
|
||||
const toolInput = toolUse.input as { [key: string]: string }
|
||||
try {
|
||||
if (toolUseContext.abortController.signal.aborted) {
|
||||
@@ -566,6 +591,15 @@ export async function* runToolUse(
|
||||
mcpServerType,
|
||||
mcpServerBaseUrl,
|
||||
)) {
|
||||
if (
|
||||
update.message?.type === 'user' &&
|
||||
Array.isArray(update.message.message.content) &&
|
||||
update.message.message.content.some(
|
||||
block => block.type === 'tool_result' && !block.is_error,
|
||||
)
|
||||
) {
|
||||
recordDoomLoopProgress(toolUseContext.doomLoopKey ?? toolUseContext.agentId)
|
||||
}
|
||||
yield update
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
+45
-8
@@ -14,6 +14,7 @@
|
||||
import { createHash } from 'crypto'
|
||||
|
||||
const DEFAULT_THRESHOLD = 3
|
||||
const MAX_CONSECUTIVE_BLOCKS = 5
|
||||
|
||||
const MAIN_AGENT_KEY = 'main'
|
||||
|
||||
@@ -23,15 +24,27 @@ interface DoomLoopState {
|
||||
lastSignature: ToolSignature | null
|
||||
consecutiveCount: number
|
||||
blocked: boolean
|
||||
consecutiveBlocks: number
|
||||
terminalMessage: string | null
|
||||
}
|
||||
|
||||
const stateByAgent = new Map<string, DoomLoopState>()
|
||||
let stateByTurn = new WeakMap<object, DoomLoopState>()
|
||||
|
||||
function getState(agentKey: string): DoomLoopState {
|
||||
let state = stateByAgent.get(agentKey)
|
||||
function getState(agentKey: string | object): DoomLoopState {
|
||||
let state = typeof agentKey === 'string'
|
||||
? stateByAgent.get(agentKey)
|
||||
: stateByTurn.get(agentKey)
|
||||
if (!state) {
|
||||
state = { lastSignature: null, consecutiveCount: 0, blocked: false }
|
||||
stateByAgent.set(agentKey, state)
|
||||
state = {
|
||||
lastSignature: null,
|
||||
consecutiveCount: 0,
|
||||
blocked: false,
|
||||
consecutiveBlocks: 0,
|
||||
terminalMessage: null,
|
||||
}
|
||||
if (typeof agentKey === 'string') stateByAgent.set(agentKey, state)
|
||||
else stateByTurn.set(agentKey, state)
|
||||
}
|
||||
return state
|
||||
}
|
||||
@@ -59,7 +72,7 @@ function computeSignature(toolName: string, input: unknown): ToolSignature {
|
||||
export function checkDoomLoop(
|
||||
toolName: string,
|
||||
input: unknown,
|
||||
options: { threshold?: number; agentKey?: string } = {},
|
||||
options: { threshold?: number; agentKey?: string | object } = {},
|
||||
): { blocked: boolean; count: number } {
|
||||
const threshold = options.threshold ?? DEFAULT_THRESHOLD
|
||||
const state = getState(options.agentKey ?? MAIN_AGENT_KEY)
|
||||
@@ -81,24 +94,48 @@ export function checkDoomLoop(
|
||||
return { blocked: state.blocked, count: state.consecutiveCount }
|
||||
}
|
||||
|
||||
/** Count actual guard blocks; latch termination so late parallel results cannot undo it. */
|
||||
export function recordDoomLoopBlock(
|
||||
toolName: string,
|
||||
lastError: string,
|
||||
agentKey: string | object = MAIN_AGENT_KEY,
|
||||
): void {
|
||||
const state = getState(agentKey)
|
||||
if (state.terminalMessage) return
|
||||
state.consecutiveBlocks++
|
||||
if (state.consecutiveBlocks >= MAX_CONSECUTIVE_BLOCKS) {
|
||||
state.terminalMessage =
|
||||
`Turn force-terminated: ${toolName} was blocked ${state.consecutiveBlocks} consecutive times by the doom-loop guard. ` +
|
||||
`Last error: ${lastError} Ask the user for help before retrying.`
|
||||
}
|
||||
}
|
||||
|
||||
/** A non-blocked call or successful result breaks the streak, until termination. */
|
||||
export function recordDoomLoopProgress(agentKey: string | object = MAIN_AGENT_KEY): void {
|
||||
const state = getState(agentKey)
|
||||
if (!state.terminalMessage) state.consecutiveBlocks = 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset doom loop state for one agent (e.g., at the start of its query turn).
|
||||
* Defaults to the main thread, matching checkDoomLoop's default key.
|
||||
*/
|
||||
export function resetDoomLoop(agentKey: string = MAIN_AGENT_KEY): void {
|
||||
stateByAgent.delete(agentKey)
|
||||
export function resetDoomLoop(agentKey: string | object = MAIN_AGENT_KEY): void {
|
||||
if (typeof agentKey === 'string') stateByAgent.delete(agentKey)
|
||||
else stateByTurn.delete(agentKey)
|
||||
}
|
||||
|
||||
/** Clear every agent's state. Test isolation only. */
|
||||
export function resetAllDoomLoops(): void {
|
||||
stateByAgent.clear()
|
||||
stateByTurn = new WeakMap()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current doom loop state for diagnostics.
|
||||
*/
|
||||
export function getDoomLoopState(
|
||||
agentKey: string = MAIN_AGENT_KEY,
|
||||
agentKey: string | object = MAIN_AGENT_KEY,
|
||||
): Readonly<DoomLoopState> {
|
||||
return { ...getState(agentKey) }
|
||||
}
|
||||
|
||||
@@ -331,7 +331,9 @@ export async function* handleOrphanedPermission(
|
||||
[finalToolUseBlock],
|
||||
[assistantMessage],
|
||||
canUseTool,
|
||||
processUserInputContext,
|
||||
// Recovery runs before query() assigns a turn identity. Keep each
|
||||
// independently approved action out of the process-wide fallback state.
|
||||
{ ...processUserInputContext, doomLoopKey: {} },
|
||||
)) {
|
||||
if (update.message) {
|
||||
mutableMessages.push(update.message)
|
||||
|
||||
Reference in New Issue
Block a user