From 870ea13117d185d7b662a50ccbe5ac3bc96caa4e Mon Sep 17 00:00:00 2001 From: JATMN Date: Tue, 27 Oct 2026 15:16:46 -0700 Subject: [PATCH] fix(query): terminate turns after consecutive doom-loop blocks (#2460) * fix(query): terminate turns after consecutive doom-loop blocks * fix(query): isolate doom-loop turns and release terminal resources * fix(query): isolate recovered approval doom-loop state * test(query): cover parallel task success after doom termination --- docs/advanced-setup.md | 8 + src/Tool.ts | 2 + src/query.ts | 47 ++- src/query/toolFailureLoopGuard.test.ts | 309 +++++++++++++++++++- src/services/tools/StreamingToolExecutor.ts | 2 + src/services/tools/toolExecution.test.ts | 51 ++++ src/services/tools/toolExecution.ts | 40 ++- src/utils/doomLoop.ts | 53 +++- src/utils/queryHelpers.ts | 4 +- 9 files changed, 497 insertions(+), 19 deletions(-) diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md index 8cb3af410..99154a6d4 100644 --- a/docs/advanced-setup.md +++ b/docs/advanced-setup.md @@ -573,6 +573,14 @@ host. Without this variable the behavior is unchanged. | `OPENCLAUDE_DISABLE_CO_AUTHORED_BY` | No | Suppress the default `Co-Authored-By` trailer in generated git commits | | `OPENCLAUDE_LOG_TOKEN_USAGE` | No | When truthy (e.g. `verbose`), emits one JSON line on stderr per API request with input/output/cache tokens and the resolved provider. **User-facing debug output** — complements the REPL display controlled by `/config showCacheStats`. Distinct from `CLAUDE_CODE_ENABLE_TOKEN_USAGE_ATTACHMENT`, which is **model-facing** (injects context usage info into the prompt itself). Both can run together. | +Repeated identical tool calls are blocked starting with the third call. If a model +ignores that feedback, five consecutive doom-loop blocks force-terminate its turn +without another model request. A non-blocked call or successful tool result resets +the consecutive-block count; each agent and new user turn tracks its own streak. +Headless runs (`--bare -p`) print the terminal error and exit with status 1. Change +the approach or tool arguments before retrying. This safeguard applies even when +the general tool-failure threshold or turn limit is disabled. + Model env vars are provider-scoped: first-party Anthropic sessions read `ANTHROPIC_MODEL`, OpenAI-compatible sessions read `OPENAI_MODEL`, Gemini reads `GEMINI_MODEL`, Gemini on Vertex reads `GEMINI_VERTEX_MODEL`, and Mistral reads diff --git a/src/Tool.ts b/src/Tool.ts index aeb8d1060..bd51b2930 100644 --- a/src/Tool.ts +++ b/src/Tool.ts @@ -290,6 +290,8 @@ export type ToolUseContext = { ) => void setConversationId?: (id: UUID) => void agentId?: AgentId // Only set for subagents; use getSessionId() for session ID. Hooks use this to distinguish subagent calls. + /** Private query-turn identity, preserved by tool-context copies. */ + doomLoopKey?: object /** Last submitted transcript for this producer; undefined for other IDs. */ getAgentTranscriptPath?: (agentId: AgentId) => string | undefined agentType?: string // Subagent type name. For the main thread's --agent type, hooks fall back to getMainThreadAgentType(). diff --git a/src/query.ts b/src/query.ts index 4fa75ee99..a1434c9ed 100644 --- a/src/query.ts +++ b/src/query.ts @@ -674,7 +674,19 @@ export async function* query( ) { params.toolUseContext.queryModelSelection = { agentId: params.toolUseContext.agentId } } - const terminal = yield* queryLoop(params, consumedCommandUuids) + // SDK sessions can query concurrently without an agentId. A fresh private + // key also separates nested/subsequent queries that inherit a tool context. + const doomLoopKey = {} + const { resetDoomLoop } = await import('./utils/doomLoop.js') + let terminal: Terminal + try { + terminal = yield* queryLoop({ + ...params, + toolUseContext: { ...params.toolUseContext, doomLoopKey }, + }, consumedCommandUuids) + } finally { + resetDoomLoop(doomLoopKey) + } // Only reached if queryLoop returned normally. Skipped on throw (error // propagates through yield*) and on .return() (Return completion closes // both generators). This gives the same asymmetric started-without-completed @@ -696,11 +708,7 @@ async function* queryLoop( | ToolUseSummaryMessage, Terminal > { - // Reset this agent's doom loop detection at the start of each query turn. - // Keyed by agentId so a subagent starting mid-turn doesn't wipe the main - // thread's counter (or a sibling agent's). - const { resetDoomLoop } = await import('./utils/doomLoop.js') - resetDoomLoop(params.toolUseContext.agentId) + const { getDoomLoopState } = await import('./utils/doomLoop.js') // Start a new turn for multi-turn context tracking if ( @@ -1620,6 +1628,10 @@ async function* queryLoop( queryTracking, queryLifecycle: toolUseContext.queryLifecycle, onProviderRequestStart: () => { + if (getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage) { + providerDispatchRejected = true + return false + } if (toolUseContext.abortController.signal.aborted) { providerDispatchRejected = true return false @@ -1862,6 +1874,7 @@ async function* queryLoop( } } if (providerDispatchRejected) { + if (getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage) break if (toolUseContext.abortController.signal.aborted) { return yield* emitAbortedStreaming( toolUseContext.abortController.signal, @@ -1903,6 +1916,9 @@ async function* queryLoop( } } } catch (innerError) { + // Streaming tools can trip the breaker before the model stream ends. + // Preserve their results and terminate instead of dispatching a fallback. + if (getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage) break if (innerError instanceof FallbackTriggeredError && fallbackModel) { // Fallback was triggered - switch model and retry currentModel = fallbackModel @@ -2063,6 +2079,18 @@ async function* queryLoop( return { reason: 'model_error', error } } + const streamingDoomTerminal = getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage + if (streamingDoomTerminal) { + if (streamingToolExecutor) { + for await (const update of streamingToolExecutor.getRemainingResults()) { + if (update.message) yield update.message + } + } + await cleanupComputerUseAtTerminal(toolUseContext) + yield createAssistantAPIErrorMessage({ content: streamingDoomTerminal }) + return { reason: 'tool_failure_loop' } + } + // Execute post-sampling hooks after model response is complete if (assistantMessages.length > 0) { void executePostSamplingHooks( @@ -2893,6 +2921,13 @@ async function* queryLoop( ) } + const doomLoopTerminal = getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage + if (doomLoopTerminal) { + await cleanupComputerUseAtTerminal(toolUseContext) + yield createAssistantAPIErrorMessage({ content: doomLoopTerminal }) + return { reason: 'tool_failure_loop' } + } + // If a hook indicated to prevent continuation, stop here if (shouldPreventContinuation) { return { reason: 'hook_stopped' } diff --git a/src/query/toolFailureLoopGuard.test.ts b/src/query/toolFailureLoopGuard.test.ts index 479ad64bd..748f6a9ad 100644 --- a/src/query/toolFailureLoopGuard.test.ts +++ b/src/query/toolFailureLoopGuard.test.ts @@ -1,10 +1,22 @@ import type { Message } from '../types/message.js' -import { expect, test } from 'bun:test' +import { expect, spyOn, test } from 'bun:test' +import * as queryConfig from './config.js' +import { FallbackTriggeredError } from '../services/api/withRetry.js' +import { acquireSharedMutationLock, releaseSharedMutationLock } from '../test/sharedMutationLock.js' +import { z } from 'zod/v4' +import { createToolFixture } from '../test/toolFixtures.js' +import { createTaskStateBase } from '../Task.js' +import type { LocalShellTaskState } from '../tasks/LocalShellTask/guards.js' +import { TaskOutputTool } from '../tools/TaskOutputTool/TaskOutputTool.js' import type { ToolUseBlock } from '@anthropic-ai/sdk/resources/index.mjs' import { query, type QueryParams } from '../query.js' import type { QueryDeps } from './deps.js' import { getMissingToolResultAbortMessage } from '../utils/abortReasons.js' +import { resetDoomLoop } from '../utils/doomLoop.js' +import { handleOrphanedPermission } from '../utils/queryHelpers.js' +import { withTimeout } from '../utils/sleep.js' +import type { ProcessUserInputContext } from '../utils/processUserInput/processUserInput.js' import { createAssistantMessage, createCompactBoundaryMessage, @@ -19,6 +31,301 @@ import { const querySourceFile = new URL('../query.ts', import.meta.url) +test('independent recovered approvals do not inherit another turn’s doom state', async () => { + resetDoomLoop() + const executedValues: number[] = [] + const resultIds: string[] = [] + const errors: string[] = [] + const tool = createToolFixture(z.object({ value: z.number() }), { + name: 'RecoveredApprovalTool', + maxResultSizeChars: 1000, + async call({ value }) { + executedValues.push(value) + return { data: 'ok' } + }, + }) + try { + for (let i = 0; i < 8; i++) { + const params = makeQueryParams(async function* () { + yield createAssistantMessage({ content: 'done' }) + } as QueryDeps['callModel'], { querySource: 'sdk' }) + params.toolUseContext.options.tools = [tool] + const context: ProcessUserInputContext = { + ...params.toolUseContext, + options: { ...params.toolUseContext.options, ideInstallationStatus: null, theme: 'dark' }, + setMessages: () => {}, + onChangeAPIKey: () => {}, + } + const id = `recovered-${i}` + const input = { value: i < 7 ? 1 : 2 } + for await (const message of handleOrphanedPermission({ + permissionResult: { behavior: 'allow', toolUseID: id, updatedInput: input }, + assistantMessage: createAssistantMessage({ content: [toolUse(id, tool.name, input)] }), + }, [tool], [], context)) { + if (message.type !== 'user' || !Array.isArray(message.message.content)) continue + for (const block of message.message.content) { + if ( + typeof block !== 'object' || block === null || + !('type' in block) || block.type !== 'tool_result' || + !('tool_use_id' in block) || typeof block.tool_use_id !== 'string' + ) continue + resultIds.push(block.tool_use_id) + if ('is_error' in block && block.is_error) errors.push(JSON.stringify(block)) + } + } + // QueryEngine continues with an ordinary query after the recovered tool. + for await (const _message of query(params)) { /* drain */ } + } + expect(executedValues).toEqual([1, 1, 1, 1, 1, 1, 1, 2]) + expect(resultIds).toEqual(Array.from({ length: 8 }, (_, i) => `recovered-${i}`)) + expect(errors).toEqual([]) + } finally { + resetDoomLoop() + } +}) + +test('concurrent SDK turns cannot reset or terminate each other', async () => { + function session(name: string) { + let calls = 0 + const tool = createToolFixture(z.object({}), { + name, + async call() { return { data: 'ok' } }, + }) + const params = makeQueryParams(async function* () { + calls++ + yield createAssistantMessage({ + content: calls <= 8 ? [toolUse(`${name}-${calls}`, name)] : 'done', + }) + } as QueryDeps['callModel'], { querySource: 'sdk' }) + params.toolUseContext.options.tools = [tool] + return { turn: query(params), calls: () => calls } + } + const first = session('FirstSessionTool') + const second = session('SecondSessionTool') + // Suspend the first turn after four blocks, while its generator still owns + // its state; run the second session all the way through terminal cleanup. + let next = await first.turn.next() + while (!next.done) { + if (first.calls() === 6 && next.value.type === 'user') break + next = await first.turn.next() + } + let secondTerminal: Message | undefined + for await (const message of second.turn) { + if (message.type === 'assistant') secondTerminal = message + } + let firstTerminal: Message | undefined + for await (const message of first.turn) { + if (message.type === 'assistant') firstTerminal = message + } + expect(first.calls()).toBe(7) + expect(second.calls()).toBe(7) + expect(JSON.stringify(firstTerminal)).toContain('FirstSessionTool was blocked 5 consecutive') + expect(JSON.stringify(secondTerminal)).toContain('SecondSessionTool was blocked 5 consecutive') +}) + +test('streaming doom termination wins over fallback and resets on unknown calls', async () => { + await acquireSharedMutationLock('doom-loop-streaming-config') + const config = queryConfig.buildQueryConfig() + const configSpy = spyOn(queryConfig, 'buildQueryConfig').mockReturnValue({ + ...config, + gates: { ...config.gates, streamingToolExecution: true }, + }) + try { + let modelCalls = 0 + const tool = createToolFixture(z.object({}), { + name: 'StreamingDoomTool', + async call() { return { data: 'ok' } }, + }) + const params = makeQueryParams(async function* () { + modelCalls++ + yield createAssistantMessage({ content: [ + toolUse(`stream-${modelCalls}`, modelCalls === 7 ? 'UnknownTool' : tool.name), + ] }) + if (modelCalls === 12) throw new FallbackTriggeredError('test-model', 'fallback') + if (modelCalls > 12) throw new Error('Unexpected fallback model request') + } as QueryDeps['callModel'], { fallbackModel: 'fallback' }) + params.toolUseContext.options.tools = [tool] + let terminal: Message | undefined + const resultIds: string[] = [] + for await (const message of query(params)) { + if (message.type === 'assistant') terminal = message + if (message.type === 'user' && Array.isArray(message.message.content)) { + for (const block of message.message.content) { + if (block.type === 'tool_result') resultIds.push(block.tool_use_id) + } + } + } + // Two allowed, four blocked, unknown (resets), then five blocked calls. + expect(modelCalls).toBe(12) + expect(resultIds).toEqual(Array.from({ length: 12 }, (_, i) => `stream-${i + 1}`)) + expect(terminal).toMatchObject({ type: 'assistant', isApiErrorMessage: true }) + expect(JSON.stringify(terminal)).toContain('force-terminated') + } finally { + configSpy.mockRestore() + releaseSharedMutationLock() + } +}) + +for (const streaming of [false, true]) { + test(`post-latch task success cannot resume execution (streaming=${streaming})`, async () => { + await acquireSharedMutationLock('doom-loop-post-latch') + const config = queryConfig.buildQueryConfig() + const configSpy = spyOn(queryConfig, 'buildQueryConfig').mockReturnValue({ + ...config, gates: { ...config.gates, streamingToolExecution: streaming }, + }) + const task: LocalShellTaskState = { + ...createTaskStateBase('bdoomlate', 'local_bash', 'pending task'), + type: 'local_bash', status: 'running', command: '', shellCommand: null, + completionStatusSentInAttachment: false, lastReportedTotalLines: 0, isBackgrounded: true, + } + let started = 0 + let markStarted!: () => void + const bothStarted = new Promise(resolve => { markStarted = resolve }) + const pending: Promise[] = [] + const taskTool = { + ...TaskOutputTool, + call(...args: Parameters) { + if (++started === 2) markStarted() + const result = TaskOutputTool.call(...args) + pending.push(result) + return result + }, + } + let extraExecutions = 0 + const afterLatchTool = createToolFixture(z.object({}), { + name: 'AfterLatchTool', + async call() { extraExecutions++; return { data: 'unexpected' } }, + }) + let modelCalls = 0 + const ids = Array.from({ length: 7 }, (_, i) => `parallel-${i + 1}`) + const params = makeQueryParams(async function* () { + modelCalls++ + yield createAssistantMessage({ content: modelCalls === 1 + ? [...ids.map(id => toolUse(id, taskTool.name, { task_id: task.id, block: true, timeout: 30000 })), + toolUse('after-latch', afterLatchTool.name)] + : 'done' }) + } as QueryDeps['callModel']) + params.toolUseContext.options.tools = [taskTool, afterLatchTool] + const getAppState = params.toolUseContext.getAppState + params.toolUseContext.getAppState = () => ({ ...getAppState(), tasks: { [task.id]: task } }) + params.toolUseContext.setAppState = () => {} + const results: { id: string; error: boolean; content: string }[] = [] + let terminal: Message | undefined + try { + for await (const message of query(params)) { + if (message.type === 'assistant') terminal = message + if (message.type !== 'user' || !Array.isArray(message.message.content)) continue + for (const block of message.message.content) { + if (block.type !== 'tool_result') continue + results.push({ id: block.tool_use_id, error: Boolean(block.is_error), content: String(block.content) }) + if (block.tool_use_id === 'parallel-7') { + expect(String(block.content)).toContain('Blocked: 7 consecutive calls') + await withTimeout(bothStarted, 2000, 'Both permitted task calls must enter their wait') + expect(task.status).toBe('running') + task.status = 'completed' + task.result = { code: 0, interrupted: false } + } + } + } + expect(started).toBe(2) + expect(results.map(result => result.id).sort()).toEqual([...ids, 'after-latch'].sort()) + for (const id of ['parallel-1', 'parallel-2']) { + expect(results.find(result => result.id === id)).toMatchObject({ error: false, content: expect.stringContaining('success') }) + } + expect(results.find(result => result.id === 'after-latch')?.error).toBe(true) + expect(extraExecutions).toBe(0) + expect(modelCalls).toBe(1) + expect(terminal).toMatchObject({ type: 'assistant', isApiErrorMessage: true }) + expect(JSON.stringify(terminal)).toContain('force-terminated') + } finally { + task.status = 'completed' + params.toolUseContext.abortController.abort() + await Promise.allSettled(pending) + configSpy.mockRestore() + releaseSharedMutationLock() + } + }) +} + +test('doom-loop blocks terminate the query after five consecutive blocks', async () => { + let modelCalls = 0 + let executions = 0 + const tool = createToolFixture(z.object({ value: z.string() }), { + name: 'DoomLoopTestTool', + async call() { + executions++ + return { data: 'ok' } + }, + }) + const params = makeQueryParams(async function* () { + modelCalls++ + // Bound the broken baseline without relying on a timeout to prove failure. + yield createAssistantMessage({ + content: modelCalls <= 8 + ? [toolUse(`repeat-${modelCalls}`, tool.name, { value: 'same' })] + : 'model finally stopped', + }) + } as QueryDeps['callModel']) + params.toolUseContext.options.tools = [tool] + const messages: Message[] = [] + const turn = query(params) + let next = await turn.next() + while (!next.done) { + if (next.value.type === 'assistant') messages.push(next.value) + next = await turn.next() + } + expect(executions).toBe(2) + expect(modelCalls).toBe(7) + expect(next.value).toEqual({ reason: 'tool_failure_loop' }) + const terminal = messages.at(-1) + expect(terminal).toMatchObject({ type: 'assistant', isApiErrorMessage: true }) + if (terminal?.type !== 'assistant') throw new Error('Missing terminal error') + expect(terminal.message.content).toEqual([ + expect.objectContaining({ + type: 'text', + text: expect.stringContaining('force-terminated'), + }), + ]) + const text = JSON.stringify(terminal.message.content) + expect(text).toContain(tool.name) + expect(text).toContain('5 consecutive') + expect(text).toContain('Blocked: 7 consecutive calls') + + // A subsequent user turn starts clean even after a latched termination. + modelCalls = 0 + for await (const _message of query({ ...params, maxTurns: 1 })) { /* drain */ } + expect(executions).toBe(3) + expect(modelCalls).toBe(1) +}) + +test('changed arguments reset doom blocks during a progressing query', async () => { + let modelCalls = 0 + let executions = 0 + const tool = createToolFixture(z.object({ value: z.number() }), { + name: 'DoomLoopProgressTool', + async call() { + executions++ + return { data: 'ok' } + }, + }) + const params = makeQueryParams(async function* () { + modelCalls++ + yield createAssistantMessage({ + content: modelCalls <= 12 + ? [toolUse(`progress-${modelCalls}`, tool.name, { + value: modelCalls <= 6 ? 1 : 2, + })] + : 'done', + }) + } as QueryDeps['callModel']) + params.toolUseContext.options.tools = [tool] + for await (const message of query(params)) { + if (message.type === 'assistant') expect(message.isApiErrorMessage).not.toBe(true) + } + expect(modelCalls).toBe(13) + expect(executions).toBe(4) +}) + function toolUse( id: string, name: string, diff --git a/src/services/tools/StreamingToolExecutor.ts b/src/services/tools/StreamingToolExecutor.ts index 087dcd01f..06ad04aaa 100644 --- a/src/services/tools/StreamingToolExecutor.ts +++ b/src/services/tools/StreamingToolExecutor.ts @@ -9,6 +9,7 @@ import { findToolByName, type Tools, type ToolUseContext } from '../../Tool.js' import { BASH_TOOL_NAME } from '../../tools/BashTool/toolName.js' import type { AssistantMessage, Message } from '../../types/message.js' import { createChildAbortController } from '../../utils/abortController.js' +import { recordDoomLoopProgress } from '../../utils/doomLoop.js' import { getMissingToolResultAbortMessage, shouldCreateUserInterruptionMessage, @@ -118,6 +119,7 @@ export class StreamingToolExecutor { addTool(block: ToolUseBlock, assistantMessage: AssistantMessage): void { const toolDefinition = findToolByName(this.toolDefinitions, block.name) if (!toolDefinition) { + recordDoomLoopProgress(this.toolUseContext.doomLoopKey ?? this.toolUseContext.agentId) this.tools.push({ id: block.id, block, diff --git a/src/services/tools/toolExecution.test.ts b/src/services/tools/toolExecution.test.ts index db2dfd1d4..c85e1e5e6 100644 --- a/src/services/tools/toolExecution.test.ts +++ b/src/services/tools/toolExecution.test.ts @@ -16,7 +16,9 @@ import { BASH_TOOL_NAME } from '../../tools/BashTool/toolName.js' import { FILE_EDIT_TOOL_NAME } from '../../tools/FileEditTool/constants.js' import { FILE_WRITE_TOOL_NAME } from '../../tools/FileWriteTool/constants.js' import { NOTEBOOK_EDIT_TOOL_NAME } from '../../tools/NotebookEditTool/constants.js' +import { asAgentId } from '../../types/ids.js' import { AbortError } from '../../utils/errors.js' +import { getDoomLoopState, resetDoomLoop } from '../../utils/doomLoop.js' import { CANCEL_MESSAGE, createAssistantMessage } from '../../utils/messages.js' import { type QueryActiveOperationSnapshot, @@ -144,6 +146,55 @@ async function collectRunToolUse( return updates } +test('a late successful tool result resets doom blocks without clearing another agent', async () => { + const agentId = asAgentId('doom-loop-late-success') + const siblingId = asAgentId('doom-loop-sibling') + let finish!: () => void + let started!: () => void + const ready = new Promise(resolve => { started = resolve }) + const pending = new Promise(resolve => { finish = resolve }) + let calls = 0 + const tool = createToolFixture(z.object({ value: z.string() }), { + name: 'DoomLoopLateSuccessTool', + async call() { + if (++calls === 2) { + started() + await pending + } + return { data: 'ok' } + }, + }) + const context = createLifecycleToolUseContext([tool], new CountingQueryLifecycleTracker()) + context.agentId = agentId + const sibling = { ...context, agentId: siblingId } + let inFlight: Promise | undefined + try { + await collectRunToolUse(tool, { value: 'same' }, context) + inFlight = collectRunToolUse(tool, { value: 'same' }, context) + await ready + for (let i = 0; i < 4; i++) await collectRunToolUse(tool, { value: 'same' }, context) + for (let i = 0; i < 6; i++) await collectRunToolUse(tool, { value: 'same' }, sibling) + expect(getDoomLoopState(agentId).consecutiveBlocks).toBe(4) + finish() + await inFlight + expect(getDoomLoopState(agentId).consecutiveBlocks).toBe(0) + expect(getDoomLoopState(siblingId).consecutiveBlocks).toBe(4) + for (let i = 0; i < 4; i++) await collectRunToolUse(tool, { value: 'same' }, context) + expect(getDoomLoopState(agentId).terminalMessage).toBeNull() + await collectRunToolUse(tool, { value: 'same' }, context) + expect(getDoomLoopState(agentId).terminalMessage).toContain('5 consecutive') + const executedBefore = calls + const skipped = await collectRunToolUse(tool, { value: 'different' }, context) + expect(calls).toBe(executedBefore) + expect(firstToolResultText(skipped[0]?.message)).toContain('force-terminated') + } finally { + finish() + await inFlight + resetDoomLoop(agentId) + resetDoomLoop(siblingId) + } +}) + describe('getSchemaValidationErrorOverride', () => { test('returns actionable missing-skill error for SkillTool', () => { expect(getSchemaValidationErrorOverride(SkillTool, {})).toBe( diff --git a/src/services/tools/toolExecution.ts b/src/services/tools/toolExecution.ts index 10183f453..cbefd37db 100644 --- a/src/services/tools/toolExecution.ts +++ b/src/services/tools/toolExecution.ts @@ -63,7 +63,12 @@ import { shouldCreateUserInterruptionMessage, } from '../../utils/abortReasons.js' import { logForDebugging } from '../../utils/debug.js' -import { checkDoomLoop } from '../../utils/doomLoop.js' +import { + checkDoomLoop, + getDoomLoopState, + recordDoomLoopBlock, + recordDoomLoopProgress, +} from '../../utils/doomLoop.js' import { AbortError, errorMessage, @@ -408,6 +413,22 @@ export async function* runToolUse( toolUseContext: ToolUseContext, ): AsyncGenerator { const toolName = toolUse.name + const terminalMessage = getDoomLoopState(toolUseContext.doomLoopKey ?? toolUseContext.agentId).terminalMessage + if (terminalMessage) { + // Preserve tool-use/result pairing without starting more work in this batch. + yield { + message: createUserMessage({ + content: [{ + type: 'tool_result', + tool_use_id: toolUse.id, + content: terminalMessage, + is_error: true, + }], + sourceToolAssistantUUID: assistantMessage.uuid, + }), + } + return + } // First try to find in the available tools (what the model sees) let tool = findToolByName(toolUseContext.options.tools, toolName) @@ -434,6 +455,7 @@ export async function* runToolUse( // Check if the tool exists if (!tool) { + recordDoomLoopProgress(toolUseContext.doomLoopKey ?? toolUseContext.agentId) const sanitizedToolName = sanitizeToolNameForAnalytics(toolName) logForDebugging(`Unknown tool ${toolName}: ${toolUse.id}`) logEvent('tengu_tool_use_error', { @@ -480,9 +502,11 @@ export async function* runToolUse( // Doom loop detection: block after N consecutive identical tool calls. // Keyed by agent so concurrent subagents don't pollute each other's counters. const doomLoop = checkDoomLoop(toolName, toolUse.input, { - agentKey: toolUseContext.agentId, + agentKey: toolUseContext.doomLoopKey ?? toolUseContext.agentId, }) if (doomLoop.blocked) { + const error = `Blocked: ${doomLoop.count} consecutive calls to this tool with identical input — you are likely in a loop. Change the input, try a different tool, or ask the user for help. A deliberate repeat is fine once something observable has changed (new output to check, a modified file, an updated instruction).` + recordDoomLoopBlock(toolName, error, toolUseContext.doomLoopKey ?? toolUseContext.agentId) logForDebugging(`Doom loop detected for ${toolName} (${doomLoop.count} consecutive identical calls)`) // Observability for tuning: a burst of these against varied tools would // indicate false positives (e.g. legitimate polling), not stuck agents. @@ -497,7 +521,7 @@ export async function* runToolUse( content: [ { type: 'tool_result', - content: `Blocked: ${doomLoop.count} consecutive calls to this tool with identical input — you are likely in a loop. Change the input, try a different tool, or ask the user for help. A deliberate repeat is fine once something observable has changed (new output to check, a modified file, an updated instruction).`, + content: `${error}`, is_error: true, tool_use_id: toolUse.id, }, @@ -509,6 +533,7 @@ export async function* runToolUse( return } + recordDoomLoopProgress(toolUseContext.doomLoopKey ?? toolUseContext.agentId) const toolInput = toolUse.input as { [key: string]: string } try { if (toolUseContext.abortController.signal.aborted) { @@ -566,6 +591,15 @@ export async function* runToolUse( mcpServerType, mcpServerBaseUrl, )) { + if ( + update.message?.type === 'user' && + Array.isArray(update.message.message.content) && + update.message.message.content.some( + block => block.type === 'tool_result' && !block.is_error, + ) + ) { + recordDoomLoopProgress(toolUseContext.doomLoopKey ?? toolUseContext.agentId) + } yield update } } catch (error) { diff --git a/src/utils/doomLoop.ts b/src/utils/doomLoop.ts index 9afe180df..c9a2aacdd 100644 --- a/src/utils/doomLoop.ts +++ b/src/utils/doomLoop.ts @@ -14,6 +14,7 @@ import { createHash } from 'crypto' const DEFAULT_THRESHOLD = 3 +const MAX_CONSECUTIVE_BLOCKS = 5 const MAIN_AGENT_KEY = 'main' @@ -23,15 +24,27 @@ interface DoomLoopState { lastSignature: ToolSignature | null consecutiveCount: number blocked: boolean + consecutiveBlocks: number + terminalMessage: string | null } const stateByAgent = new Map() +let stateByTurn = new WeakMap() -function getState(agentKey: string): DoomLoopState { - let state = stateByAgent.get(agentKey) +function getState(agentKey: string | object): DoomLoopState { + let state = typeof agentKey === 'string' + ? stateByAgent.get(agentKey) + : stateByTurn.get(agentKey) if (!state) { - state = { lastSignature: null, consecutiveCount: 0, blocked: false } - stateByAgent.set(agentKey, state) + state = { + lastSignature: null, + consecutiveCount: 0, + blocked: false, + consecutiveBlocks: 0, + terminalMessage: null, + } + if (typeof agentKey === 'string') stateByAgent.set(agentKey, state) + else stateByTurn.set(agentKey, state) } return state } @@ -59,7 +72,7 @@ function computeSignature(toolName: string, input: unknown): ToolSignature { export function checkDoomLoop( toolName: string, input: unknown, - options: { threshold?: number; agentKey?: string } = {}, + options: { threshold?: number; agentKey?: string | object } = {}, ): { blocked: boolean; count: number } { const threshold = options.threshold ?? DEFAULT_THRESHOLD const state = getState(options.agentKey ?? MAIN_AGENT_KEY) @@ -81,24 +94,48 @@ export function checkDoomLoop( return { blocked: state.blocked, count: state.consecutiveCount } } +/** Count actual guard blocks; latch termination so late parallel results cannot undo it. */ +export function recordDoomLoopBlock( + toolName: string, + lastError: string, + agentKey: string | object = MAIN_AGENT_KEY, +): void { + const state = getState(agentKey) + if (state.terminalMessage) return + state.consecutiveBlocks++ + if (state.consecutiveBlocks >= MAX_CONSECUTIVE_BLOCKS) { + state.terminalMessage = + `Turn force-terminated: ${toolName} was blocked ${state.consecutiveBlocks} consecutive times by the doom-loop guard. ` + + `Last error: ${lastError} Ask the user for help before retrying.` + } +} + +/** A non-blocked call or successful result breaks the streak, until termination. */ +export function recordDoomLoopProgress(agentKey: string | object = MAIN_AGENT_KEY): void { + const state = getState(agentKey) + if (!state.terminalMessage) state.consecutiveBlocks = 0 +} + /** * Reset doom loop state for one agent (e.g., at the start of its query turn). * Defaults to the main thread, matching checkDoomLoop's default key. */ -export function resetDoomLoop(agentKey: string = MAIN_AGENT_KEY): void { - stateByAgent.delete(agentKey) +export function resetDoomLoop(agentKey: string | object = MAIN_AGENT_KEY): void { + if (typeof agentKey === 'string') stateByAgent.delete(agentKey) + else stateByTurn.delete(agentKey) } /** Clear every agent's state. Test isolation only. */ export function resetAllDoomLoops(): void { stateByAgent.clear() + stateByTurn = new WeakMap() } /** * Get current doom loop state for diagnostics. */ export function getDoomLoopState( - agentKey: string = MAIN_AGENT_KEY, + agentKey: string | object = MAIN_AGENT_KEY, ): Readonly { return { ...getState(agentKey) } } diff --git a/src/utils/queryHelpers.ts b/src/utils/queryHelpers.ts index 59ce1ebd8..2496f55a0 100644 --- a/src/utils/queryHelpers.ts +++ b/src/utils/queryHelpers.ts @@ -331,7 +331,9 @@ export async function* handleOrphanedPermission( [finalToolUseBlock], [assistantMessage], canUseTool, - processUserInputContext, + // Recovery runs before query() assigns a turn identity. Keep each + // independently approved action out of the process-wide fallback state. + { ...processUserInputContext, doomLoopKey: {} }, )) { if (update.message) { mutableMessages.push(update.message)