mirror of
https://github.com/Gitlawb/openclaude.git
synced 2026-08-24 02:34:15 -05:00
diagnostics(query): trace interruption causality (#2111)
* diagnostics(issue-1830): trace interruption causality * test(issue-1830): lock interruption ownership matrix * fix(codex): preserve stream deadline contract * fix(diagnostics): harden interruption trace lifecycle Refs #1830 * fix(diagnostics): harden interruption trace settlement Refs #1830 * fix(diagnostics): preserve interruption causality * fix(diagnostics): address interruption trace review * fix(diagnostics): preserve tracing observer contracts * fix(diagnostics): preserve interruption trace contracts * test(permissions): cover interactive hook interrupts
This commit is contained in:
@@ -481,6 +481,8 @@ host. Without this variable the behavior is unchanged.
|
||||
| `OPENCLAUDE_MAX_TURNS` | No | Per-prompt **local** interactive REPL turn cap for the in-process query loop. Defaults to `50`. Set a larger positive integer for long autonomous local interactive sessions (for example models that take many small tool steps). CLI `--max-turns 0` explicitly disables this cap and prints a cautionary warning. Precedence for a valid override: CLI `--max-turns` → this env var → legacy `CLAUDE_CODE_MAX_TURNS` (only when this var is unset/empty) → `/config` → Max turns (interactive) → `50`. If this env var is set but invalid (zero, negative, non-integer), the default `50` is used and lower layers are not consulted — same pattern as `OPENCLAUDE_MAX_RETRIES`. Does not apply to remote-backed interactive sessions (`connect` / `ssh` / `--remote`). |
|
||||
| `OPENCLAUDE_RETRY_DELAY_MS` | No | Base retry delay in milliseconds for APIs that do not send `Retry-After`; exponential backoff starts from this value, capped at 60000 (default: 500) |
|
||||
| `OPENCLAUDE_QUERY_HARD_MAX_MS` | No | Foreground query hard maximum in milliseconds. Defaults to 1800000 (30 minutes). Use a larger positive integer for long autonomous sessions; invalid, zero, negative, fractional, or timer-overflow values are ignored with a warning. |
|
||||
| `OPENCLAUDE_INTERRUPT_TRACE` | No | Set to `1` or `true` to retain a bounded, privacy-safe interruption lifecycle trace in memory. Disabled by default. The trace contains only allowlisted lifecycle metadata—never prompts, responses, tool arguments, credentials, or raw error messages. |
|
||||
| `OPENCLAUDE_INTERRUPT_TRACE_FILE` | No | Optional absolute JSONL output path used only when `OPENCLAUDE_INTERRUPT_TRACE` is enabled. On Linux, missing parent directories are created privately and every parent is opened through `/proc/self/fd` without following symbolic links before the final regular file is appended. If the file already exists, its mode is reset to `0600` on every append, so do not configure a shared file. Other platforms retain the bounded trace in memory but do not write this file because Node does not expose an equivalent safe descriptor-relative traversal API there. Writes are best-effort and never change request behavior. Use a separate path per OpenClaude process and keep the resulting diagnostic file private. |
|
||||
| `OPENCLAUDE_DISABLE_CO_AUTHORED_BY` | No | Suppress the default `Co-Authored-By` trailer in generated git commits |
|
||||
| `OPENCLAUDE_LOG_TOKEN_USAGE` | No | When truthy (e.g. `verbose`), emits one JSON line on stderr per API request with input/output/cache tokens and the resolved provider. **User-facing debug output** — complements the REPL display controlled by `/config showCacheStats`. Distinct from `CLAUDE_CODE_ENABLE_TOKEN_USAGE_ATTACHMENT`, which is **model-facing** (injects context usage info into the prompt itself). Both can run together. |
|
||||
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from './test/sharedMutationLock.js'
|
||||
import { QueryEngine } from './QueryEngine.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
registerInterruptionController,
|
||||
} from './utils/interruptionTrace.js'
|
||||
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock('QueryEngine.interruptionTrace.test.ts')
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
} finally {
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
describe('QueryEngine interruption tracing', () => {
|
||||
test('does not record lifecycle entries while tracing is disabled', async () => {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
const engine = Object.create(QueryEngine.prototype) as QueryEngine
|
||||
const controller = new AbortController()
|
||||
;(engine as unknown as { abortController: AbortController }).abortController =
|
||||
controller
|
||||
;(engine as unknown as {
|
||||
submitMessageImpl(): AsyncGenerator<never, void, unknown>
|
||||
}).submitMessageImpl = async function* () {}
|
||||
|
||||
for await (const _message of engine.submitMessage('hello')) {
|
||||
// The stub deliberately yields nothing.
|
||||
}
|
||||
|
||||
expect(__getInterruptionTraceSnapshotForTests()).toEqual([])
|
||||
})
|
||||
|
||||
test('records a programmatic query-root interruption before aborting', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const controller = new AbortController()
|
||||
const engine = Object.create(QueryEngine.prototype) as QueryEngine
|
||||
;(engine as unknown as {
|
||||
abortController: AbortController
|
||||
}).abortController = controller
|
||||
|
||||
engine.interrupt('sdk_interrupt')
|
||||
|
||||
const requested = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'abort.requested',
|
||||
)
|
||||
expect(controller.signal.aborted).toBe(true)
|
||||
expect(requested).toMatchObject({
|
||||
source: 'sdk_interrupt',
|
||||
subsystem: 'query_engine',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
})
|
||||
|
||||
test('records start and terminal lifecycle for successful SDK turns', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const engine = Object.create(QueryEngine.prototype) as QueryEngine
|
||||
const controller = new AbortController()
|
||||
;(engine as unknown as { abortController: AbortController }).abortController =
|
||||
controller
|
||||
;(engine as unknown as {
|
||||
submitMessageImpl(): AsyncGenerator<never, void, unknown>
|
||||
}).submitMessageImpl = async function* () {}
|
||||
|
||||
for await (const _message of engine.submitMessage('hello')) {
|
||||
// The stub deliberately yields nothing.
|
||||
}
|
||||
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const started = trace.find(entry => entry.event === 'query.started')
|
||||
const terminal = trace.find(entry => entry.event === 'query.terminal')
|
||||
expect(started).toMatchObject({
|
||||
subsystem: 'query_engine',
|
||||
querySource: 'sdk',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
expect(terminal).toMatchObject({
|
||||
subsystem: 'query_engine',
|
||||
queryId: started?.queryId,
|
||||
outcome: 'completed',
|
||||
})
|
||||
expect(typeof started?.eventId).toBe('string')
|
||||
expect(typeof terminal?.causalEventId).toBe('string')
|
||||
expect(terminal!.causalEventId).toBe(started!.eventId)
|
||||
})
|
||||
|
||||
test('records aborted and failed SDK turn terminals', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
|
||||
for (const scenario of ['aborted', 'failed'] as const) {
|
||||
__resetInterruptionTraceForTests()
|
||||
const engine = Object.create(QueryEngine.prototype) as QueryEngine
|
||||
const controller = new AbortController()
|
||||
;(engine as unknown as { abortController: AbortController }).abortController =
|
||||
controller
|
||||
;(engine as unknown as {
|
||||
submitMessageImpl(): AsyncGenerator<never, void, unknown>
|
||||
}).submitMessageImpl = async function* () {
|
||||
if (scenario === 'aborted') {
|
||||
controller.abort('interrupt')
|
||||
return
|
||||
}
|
||||
throw new Error('turn failed')
|
||||
}
|
||||
|
||||
const drain = async () => {
|
||||
for await (const _message of engine.submitMessage('hello')) {
|
||||
// The stub deliberately yields nothing.
|
||||
}
|
||||
}
|
||||
if (scenario === 'failed') await expect(drain()).rejects.toThrow('turn failed')
|
||||
else await drain()
|
||||
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const started = trace.find(entry => entry.event === 'query.started')
|
||||
const terminal = trace.find(entry => entry.event === 'query.terminal')
|
||||
expect(terminal?.outcome).toBe(scenario)
|
||||
expect(typeof started?.eventId).toBe('string')
|
||||
expect(typeof terminal?.eventId).toBe('string')
|
||||
if (scenario === 'aborted') {
|
||||
const observed = trace.find(
|
||||
entry => entry.event === 'signal.observed',
|
||||
)
|
||||
expect(typeof observed?.eventId).toBe('string')
|
||||
expect(terminal?.causalEventId).toBe(observed!.eventId)
|
||||
} else {
|
||||
expect(terminal?.causalEventId).toBe(started!.eventId)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('registers the query root when tracing is enabled at the turn boundary', async () => {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
const engine = Object.create(QueryEngine.prototype) as QueryEngine
|
||||
const controller = new AbortController()
|
||||
;(engine as unknown as { abortController: AbortController }).abortController =
|
||||
controller
|
||||
registerInterruptionController(controller, {
|
||||
subsystem: 'query_engine',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
;(engine as unknown as {
|
||||
submitMessageImpl(): AsyncGenerator<never, void, unknown>
|
||||
}).submitMessageImpl = async function* () {
|
||||
controller.abort()
|
||||
}
|
||||
|
||||
for await (const _message of engine.submitMessage('hello')) {
|
||||
// The stub deliberately yields nothing.
|
||||
}
|
||||
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const registered = trace.find(
|
||||
entry =>
|
||||
entry.event === 'controller.registered' &&
|
||||
entry.controllerRole === 'query-root',
|
||||
)
|
||||
const observed = trace.find(entry => entry.event === 'signal.observed')
|
||||
const terminal = trace.find(entry => entry.event === 'query.terminal')
|
||||
expect(registered).toBeDefined()
|
||||
expect(typeof observed?.eventId).toBe('string')
|
||||
expect(terminal).toMatchObject({
|
||||
outcome: 'aborted',
|
||||
causalEventId: observed!.eventId,
|
||||
})
|
||||
})
|
||||
})
|
||||
+68
-2
@@ -45,6 +45,14 @@ import { SYNTHETIC_OUTPUT_TOOL_NAME } from './tools/SyntheticOutputTool/Syntheti
|
||||
import type { Message } from './types/message.js'
|
||||
import type { OrphanedPermission } from './types/textInputTypes.js'
|
||||
import { createAbortController } from './utils/abortController.js'
|
||||
import {
|
||||
flushInterruptionTrace,
|
||||
getInterruptionSignalAbortEventId,
|
||||
isInterruptionTraceEnabled,
|
||||
registerInterruptionController,
|
||||
requestAbort,
|
||||
traceInterruptionEvent,
|
||||
} from './utils/interruptionTrace.js'
|
||||
import { validateArrayOf, assertNonEmptyString, assertObject, assertFunction } from './utils/validation.js'
|
||||
import { invalidateRemovedToolSchemas } from './utils/toolSchemaCache.js'
|
||||
import type { AttributionState } from './utils/commitAttribution.js'
|
||||
@@ -205,6 +213,10 @@ export class QueryEngine {
|
||||
this.config = config
|
||||
this.mutableMessages = config.initialMessages ?? []
|
||||
this.abortController = config.abortController ?? createAbortController()
|
||||
registerInterruptionController(this.abortController, {
|
||||
subsystem: 'query_engine',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
this.permissionDenials = []
|
||||
this.readFileState = config.readFileCache
|
||||
this.totalUsage = EMPTY_USAGE
|
||||
@@ -213,6 +225,56 @@ export class QueryEngine {
|
||||
async *submitMessage(
|
||||
prompt: string | ContentBlockParam[],
|
||||
options?: { uuid?: string; isMeta?: boolean },
|
||||
): AsyncGenerator<SDKMessage, void, unknown> {
|
||||
const queryId = isInterruptionTraceEnabled() ? randomUUID() : undefined
|
||||
registerInterruptionController(this.abortController, {
|
||||
subsystem: 'query_engine',
|
||||
controllerRole: 'query-root',
|
||||
queryId,
|
||||
querySource: 'sdk',
|
||||
}, { refreshQueryContext: true })
|
||||
const startedEventId = traceInterruptionEvent('query.started', {
|
||||
subsystem: 'query_engine',
|
||||
phase: 'running',
|
||||
queryId,
|
||||
querySource: 'sdk',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
let outcome = 'consumer_closed'
|
||||
let terminalError: unknown
|
||||
try {
|
||||
yield* this.submitMessageImpl(prompt, options)
|
||||
outcome = this.abortController.signal.aborted ? 'aborted' : 'completed'
|
||||
} catch (error) {
|
||||
terminalError = error
|
||||
outcome = this.abortController.signal.aborted ? 'aborted' : 'failed'
|
||||
throw error
|
||||
} finally {
|
||||
const terminalOutcome = this.abortController.signal.aborted
|
||||
? 'aborted'
|
||||
: outcome
|
||||
traceInterruptionEvent('query.terminal', {
|
||||
subsystem: 'query_engine',
|
||||
phase: terminalOutcome,
|
||||
queryId,
|
||||
querySource: 'sdk',
|
||||
controllerRole: 'query-root',
|
||||
outcome: terminalOutcome,
|
||||
reason: this.abortController.signal.reason,
|
||||
error: terminalError,
|
||||
causalEventId: terminalOutcome === 'aborted'
|
||||
? getInterruptionSignalAbortEventId(this.abortController.signal)
|
||||
: startedEventId,
|
||||
})
|
||||
if (this.abortController.signal.aborted) {
|
||||
flushInterruptionTrace('query_terminal')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async *submitMessageImpl(
|
||||
prompt: string | ContentBlockParam[],
|
||||
options?: { uuid?: string; isMeta?: boolean },
|
||||
): AsyncGenerator<SDKMessage, void, unknown> {
|
||||
const {
|
||||
cwd,
|
||||
@@ -1212,8 +1274,12 @@ export class QueryEngine {
|
||||
}
|
||||
}
|
||||
|
||||
interrupt(): void {
|
||||
this.abortController.abort()
|
||||
interrupt(source = 'programmatic_interrupt'): void {
|
||||
requestAbort(this.abortController, undefined, {
|
||||
source,
|
||||
subsystem: 'query_engine',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
|
||||
getMessages(): readonly Message[] {
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../test/sharedMutationLock.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
} from '../utils/interruptionTrace.js'
|
||||
import {
|
||||
abortPrintModeControlRequest,
|
||||
type PrintModeControlAbortSource,
|
||||
} from './printInterruption.js'
|
||||
|
||||
const originalInterruptionTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
let hasSharedMutationLock = false
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock('cli/print.interruptionTrace.test.ts')
|
||||
hasSharedMutationLock = true
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalInterruptionTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalInterruptionTrace
|
||||
}
|
||||
} finally {
|
||||
if (hasSharedMutationLock) {
|
||||
releaseSharedMutationLock()
|
||||
hasSharedMutationLock = false
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
describe('print-mode interruption tracing', () => {
|
||||
test.each([
|
||||
['sdk_control_interrupt', 'interrupt'],
|
||||
['sdk_end_session', undefined],
|
||||
] as const)(
|
||||
'links %s input to the query and speculation aborts',
|
||||
(source: PrintModeControlAbortSource, queryReason: unknown) => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const queryController = new AbortController()
|
||||
const suggestionController = new AbortController()
|
||||
|
||||
const causalEventId = abortPrintModeControlRequest(
|
||||
queryController,
|
||||
suggestionController,
|
||||
source,
|
||||
queryReason,
|
||||
)
|
||||
|
||||
expect(queryController.signal.aborted).toBe(true)
|
||||
expect(suggestionController.signal.aborted).toBe(true)
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
expect(trace.find(entry => entry.eventId === causalEventId)).toMatchObject({
|
||||
event: `input.${source}`,
|
||||
source,
|
||||
subsystem: 'print_mode',
|
||||
})
|
||||
expect(
|
||||
trace.find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.controllerRole === 'query-root',
|
||||
),
|
||||
).toMatchObject({ source, causalEventId, subsystem: 'print_mode' })
|
||||
expect(
|
||||
trace.find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.controllerRole === 'speculation',
|
||||
),
|
||||
).toMatchObject({
|
||||
source,
|
||||
causalEventId,
|
||||
subsystem: 'prompt_suggestion',
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
test('preserves native abort behavior when tracing is disabled', () => {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
__resetInterruptionTraceForTests()
|
||||
const queryController = new AbortController()
|
||||
const suggestionController = new AbortController()
|
||||
|
||||
const causalEventId = abortPrintModeControlRequest(
|
||||
queryController,
|
||||
suggestionController,
|
||||
'sdk_control_interrupt',
|
||||
'interrupt',
|
||||
)
|
||||
|
||||
expect(causalEventId).toBeUndefined()
|
||||
expect(queryController.signal.reason).toBe('interrupt')
|
||||
expect(suggestionController.signal.reason).toBeInstanceOf(DOMException)
|
||||
expect(suggestionController.signal.reason.name).toBe('AbortError')
|
||||
expect(__getInterruptionTraceSnapshotForTests()).toEqual([])
|
||||
})
|
||||
})
|
||||
+32
-13
@@ -148,6 +148,10 @@ import {
|
||||
permissionPromptToolResultToPermissionDecision,
|
||||
} from 'src/utils/permissions/PermissionPromptToolResultSchema.js'
|
||||
import { createAbortController } from 'src/utils/abortController.js'
|
||||
import {
|
||||
registerInterruptionController,
|
||||
requestAbort,
|
||||
} from 'src/utils/interruptionTrace.js'
|
||||
import { createCombinedAbortSignal } from 'src/utils/combinedAbortSignal.js'
|
||||
import {
|
||||
generateSessionTitle,
|
||||
@@ -348,6 +352,7 @@ import {
|
||||
type HeadlessHeartbeatEvent,
|
||||
type HeadlessHeartbeatState,
|
||||
} from './headlessHeartbeat.js'
|
||||
import { abortPrintModeControlRequest } from './printInterruption.js'
|
||||
import {
|
||||
isTeamLead,
|
||||
hasActiveInProcessTeammates,
|
||||
@@ -1204,6 +1209,14 @@ function runHeadlessStreaming(
|
||||
let shutdownPromptInjected = false
|
||||
let heldBackResult: StdoutMessage | null = null
|
||||
let abortController: AbortController | undefined
|
||||
const abortActiveQuery = (source: string, reason: unknown): void => {
|
||||
if (!abortController || abortController.signal.aborted) return
|
||||
requestAbort(abortController, reason, {
|
||||
source,
|
||||
subsystem: 'print_mode',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
// Same queue sendRequest() enqueues to — one FIFO for everything.
|
||||
const output = structuredIO.outbound
|
||||
|
||||
@@ -1213,9 +1226,7 @@ function runHeadlessStreaming(
|
||||
const sigintHandler = () => {
|
||||
logForDiagnosticsNoPII('info', 'shutdown_signal', { signal: 'SIGINT' })
|
||||
options.heartbeat?.setPhase('shutting_down')
|
||||
if (abortController && !abortController.signal.aborted) {
|
||||
abortController.abort('interrupt')
|
||||
}
|
||||
abortActiveQuery('print_sigint', 'interrupt')
|
||||
void gracefulShutdown(0)
|
||||
}
|
||||
process.on('SIGINT', sigintHandler)
|
||||
@@ -2044,7 +2055,7 @@ function runHeadlessStreaming(
|
||||
// Abort the current operation when a 'now' priority message arrives.
|
||||
subscribeToCommandQueue(() => {
|
||||
if (abortController && getCommandsByMaxPriority('now').length > 0) {
|
||||
abortController.abort('interrupt')
|
||||
abortActiveQuery('priority_now', 'interrupt')
|
||||
}
|
||||
})
|
||||
|
||||
@@ -2318,6 +2329,10 @@ function runHeadlessStreaming(
|
||||
}
|
||||
|
||||
abortController = createAbortController()
|
||||
registerInterruptionController(abortController, {
|
||||
subsystem: 'print_mode',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
const turnStartTime = feature('FILE_PERSISTENCE')
|
||||
? Date.now()
|
||||
: undefined
|
||||
@@ -3029,10 +3044,12 @@ function runHeadlessStreaming(
|
||||
},
|
||||
}))
|
||||
}
|
||||
if (abortController) {
|
||||
abortController.abort('interrupt')
|
||||
}
|
||||
suggestionState.abortController?.abort()
|
||||
abortPrintModeControlRequest(
|
||||
abortController,
|
||||
suggestionState.abortController,
|
||||
'sdk_control_interrupt',
|
||||
'interrupt',
|
||||
)
|
||||
suggestionState.abortController = null
|
||||
suggestionState.lastEmitted = null
|
||||
suggestionState.pendingSuggestion = null
|
||||
@@ -3041,10 +3058,12 @@ function runHeadlessStreaming(
|
||||
logForDebugging(
|
||||
`[print.ts] end_session received, reason=${message.request.reason ?? 'unspecified'}`,
|
||||
)
|
||||
if (abortController) {
|
||||
abortController.abort()
|
||||
}
|
||||
suggestionState.abortController?.abort()
|
||||
abortPrintModeControlRequest(
|
||||
abortController,
|
||||
suggestionState.abortController,
|
||||
'sdk_end_session',
|
||||
undefined,
|
||||
)
|
||||
suggestionState.abortController = null
|
||||
suggestionState.lastEmitted = null
|
||||
suggestionState.pendingSuggestion = null
|
||||
@@ -4127,7 +4146,7 @@ function runHeadlessStreaming(
|
||||
structuredIO.injectControlResponse(response)
|
||||
},
|
||||
onInterrupt() {
|
||||
abortController?.abort('interrupt')
|
||||
abortActiveQuery('bridge_interrupt', 'interrupt')
|
||||
},
|
||||
onSetModel(model) {
|
||||
const resolved =
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import {
|
||||
requestAbort,
|
||||
traceInterruptionEvent,
|
||||
} from '../utils/interruptionTrace.js'
|
||||
|
||||
export type PrintModeControlAbortSource =
|
||||
| 'sdk_control_interrupt'
|
||||
| 'sdk_end_session'
|
||||
|
||||
export function abortPrintModeControlRequest(
|
||||
queryController: AbortController | undefined,
|
||||
suggestionController: AbortController | null,
|
||||
source: PrintModeControlAbortSource,
|
||||
queryReason: unknown,
|
||||
): string | undefined {
|
||||
const causalEventId = traceInterruptionEvent(`input.${source}`, {
|
||||
source,
|
||||
subsystem: 'print_mode',
|
||||
})
|
||||
if (queryController && !queryController.signal.aborted) {
|
||||
requestAbort(queryController, queryReason, {
|
||||
source,
|
||||
causalEventId,
|
||||
subsystem: 'print_mode',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
if (suggestionController && !suggestionController.signal.aborted) {
|
||||
requestAbort(suggestionController, undefined, {
|
||||
source,
|
||||
causalEventId,
|
||||
subsystem: 'prompt_suggestion',
|
||||
controllerRole: 'speculation',
|
||||
})
|
||||
}
|
||||
return causalEventId
|
||||
}
|
||||
+27
-1
@@ -34,6 +34,11 @@ import type {
|
||||
} from '../../../utils/permissions/PermissionResult.js'
|
||||
import type { PermissionUpdate } from '../../../utils/permissions/PermissionUpdateSchema.js'
|
||||
import type { ToolUseConfirm } from '../PermissionRequest.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
} from '../../../utils/interruptionTrace.js'
|
||||
|
||||
function createTestStreams(): {
|
||||
stdout: PassThrough
|
||||
@@ -344,6 +349,8 @@ async function renderMonitorPermission(
|
||||
}
|
||||
}
|
||||
|
||||
const originalInterruptionTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock(
|
||||
'components/permissions/MonitorPermissionRequest.test.tsx',
|
||||
@@ -353,9 +360,16 @@ beforeEach(async () => {
|
||||
}))
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
afterEach(async () => {
|
||||
try {
|
||||
mock.restore()
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalInterruptionTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalInterruptionTrace
|
||||
}
|
||||
} finally {
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
@@ -486,6 +500,8 @@ describe('MonitorPermissionRequest', () => {
|
||||
})
|
||||
|
||||
test('escape cancels the pending permission request and closes the dialog', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const onDone = mock(() => {})
|
||||
const onReject = mock(() => {})
|
||||
const { toolUseConfirm, toolUseContext, decisionPromise } =
|
||||
@@ -504,6 +520,16 @@ describe('MonitorPermissionRequest', () => {
|
||||
expect(decision.behavior).toBe('ask')
|
||||
expect(toolUseContext.abortController.signal.aborted).toBe(true)
|
||||
expect(onReject).toHaveBeenCalledTimes(1)
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.source === 'permission_abort',
|
||||
),
|
||||
).toMatchObject({
|
||||
subsystem: 'tool_permission',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
} finally {
|
||||
mounted.cleanup()
|
||||
}
|
||||
|
||||
@@ -120,7 +120,7 @@ export type ToolUseConfirm<Input extends AnyObject = AnyObject> = {
|
||||
classifierMatchedRule?: string;
|
||||
workerBadge?: WorkerBadgeProps;
|
||||
onUserInteraction(): void;
|
||||
onAbort(): void;
|
||||
onAbort(source?: string, causalEventId?: string): void;
|
||||
onDismissCheckmark?(): void;
|
||||
onAllow(updatedInput: z.infer<Input>, permissionUpdates: PermissionUpdate[], feedback?: string, contentBlocks?: ContentBlockParam[]): void;
|
||||
onReject(feedback?: string, contentBlocks?: ContentBlockParam[]): void;
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import { requestAbort } from '../../utils/interruptionTrace.js'
|
||||
|
||||
export function requestSdkRootAbort(
|
||||
controller: AbortController,
|
||||
source: string,
|
||||
subsystem: 'sdk_query' | 'sdk_session',
|
||||
): void {
|
||||
if (controller.signal.aborted) return
|
||||
requestAbort(controller, undefined, {
|
||||
source,
|
||||
subsystem,
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
} from '../../Tool.js'
|
||||
import { assembleToolPool, getTools } from '../../tools.js'
|
||||
import { createFileStateCacheWithSizeLimit } from '../../utils/fileStateCache.js'
|
||||
import { requestSdkRootAbort } from './interruption.js'
|
||||
import { init } from '../init.js'
|
||||
import {
|
||||
resolveSessionFilePath,
|
||||
@@ -756,8 +757,9 @@ class QueryImpl implements Query {
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.interrupt()
|
||||
this.abortController.abort()
|
||||
const wrapperController = this.abortController
|
||||
this.interruptWithSource('sdk_close')
|
||||
requestSdkRootAbort(wrapperController, 'sdk_close', 'sdk_query')
|
||||
// Disconnect MCP clients to prevent resource leaks
|
||||
const mcpClients = this._engine?.getMcpClients?.() ?? []
|
||||
for (const client of mcpClients) {
|
||||
@@ -773,8 +775,14 @@ class QueryImpl implements Query {
|
||||
}
|
||||
|
||||
interrupt(): void {
|
||||
this.interruptWithSource('sdk_interrupt')
|
||||
}
|
||||
|
||||
private interruptWithSource(source: string): void {
|
||||
if (this._engine) {
|
||||
this._engine.interrupt()
|
||||
this._engine.interrupt(source)
|
||||
} else {
|
||||
requestSdkRootAbort(this.abortController, source, 'sdk_query')
|
||||
}
|
||||
// Deny all pending permission prompts before clearing
|
||||
for (const [toolUseId, pending] of this.pendingPermissionPrompts) {
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
} from '../../Tool.js'
|
||||
import { assembleToolPool, getTools } from '../../tools.js'
|
||||
import { createFileStateCacheWithSizeLimit } from '../../utils/fileStateCache.js'
|
||||
import { requestSdkRootAbort } from './interruption.js'
|
||||
import { init } from '../init.js'
|
||||
import {
|
||||
resolveSessionFilePath,
|
||||
@@ -387,8 +388,14 @@ class SDKSessionImpl implements SDKSession {
|
||||
}
|
||||
|
||||
interrupt(): void {
|
||||
this.interruptWithSource('sdk_interrupt')
|
||||
}
|
||||
|
||||
private interruptWithSource(source: string): void {
|
||||
if (this._engine) {
|
||||
this._engine.interrupt()
|
||||
this._engine.interrupt(source)
|
||||
} else if (this._abortController) {
|
||||
requestSdkRootAbort(this._abortController, source, 'sdk_session')
|
||||
}
|
||||
// Deny all pending permission prompts before clearing
|
||||
for (const [toolUseId, pending] of this.pendingPermissionPrompts) {
|
||||
@@ -403,10 +410,11 @@ class SDKSessionImpl implements SDKSession {
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.interrupt()
|
||||
// Abort the AbortController to cancel any in-flight HTTP requests or
|
||||
// async operations tied to the signal. Mirrors QueryImpl.close().
|
||||
this._abortController?.abort()
|
||||
const wrapperController = this._abortController
|
||||
this.interruptWithSource('sdk_close')
|
||||
if (wrapperController) {
|
||||
requestSdkRootAbort(wrapperController, 'sdk_close', 'sdk_session')
|
||||
}
|
||||
this._abortController = null
|
||||
// Disconnect MCP clients to prevent resource leaks
|
||||
const mcpClients = this._engine?.getMcpClients?.() ?? []
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { expect, spyOn, test } from 'bun:test'
|
||||
import { QueryEngine } from '../QueryEngine.js'
|
||||
import { GrpcServer } from './server.js'
|
||||
|
||||
class FakeCall extends EventEmitter {
|
||||
writes: unknown[] = []
|
||||
ended = false
|
||||
|
||||
write(value: unknown): void {
|
||||
this.writes.push(value)
|
||||
}
|
||||
|
||||
end(): void {
|
||||
this.ended = true
|
||||
}
|
||||
}
|
||||
|
||||
async function exerciseInterruption(
|
||||
event: 'cancel' | 'end',
|
||||
): Promise<string | undefined> {
|
||||
let releaseSubmit!: () => void
|
||||
const submitBlocked = new Promise<void>(resolve => {
|
||||
releaseSubmit = resolve
|
||||
})
|
||||
let submitEntered = false
|
||||
const submitMessage = spyOn(
|
||||
QueryEngine.prototype,
|
||||
'submitMessage',
|
||||
).mockImplementation(
|
||||
async function* () {
|
||||
submitEntered = true
|
||||
await submitBlocked
|
||||
},
|
||||
)
|
||||
const interrupt = spyOn(QueryEngine.prototype, 'interrupt')
|
||||
try {
|
||||
const server = new GrpcServer()
|
||||
const call = new FakeCall()
|
||||
;(server as unknown as {
|
||||
handleChat(call: FakeCall): void
|
||||
}).handleChat(call)
|
||||
call.emit('data', {
|
||||
request: {
|
||||
message: 'test',
|
||||
working_directory: process.cwd(),
|
||||
model: 'sonnet',
|
||||
},
|
||||
})
|
||||
let interruptionEmitted = false
|
||||
for (
|
||||
let attempts = 0;
|
||||
attempts < 100 && interrupt.mock.calls.length === 0;
|
||||
attempts++
|
||||
) {
|
||||
if (event === 'cancel' && submitEntered && !interruptionEmitted) {
|
||||
call.emit('data', { cancel: {} })
|
||||
interruptionEmitted = true
|
||||
}
|
||||
if (event === 'end' && submitEntered && !interruptionEmitted) {
|
||||
call.emit('end')
|
||||
interruptionEmitted = true
|
||||
}
|
||||
await Bun.sleep(10)
|
||||
}
|
||||
if (!submitEntered) {
|
||||
throw new Error('submitMessage was never entered')
|
||||
}
|
||||
if (interrupt.mock.calls.length === 0) {
|
||||
throw new Error(
|
||||
`Timed out waiting for QueryEngine.interrupt after the '${event}' event`,
|
||||
)
|
||||
}
|
||||
return interrupt.mock.calls[0]?.[0]
|
||||
} finally {
|
||||
releaseSubmit()
|
||||
await Bun.sleep(10)
|
||||
interrupt.mockRestore()
|
||||
submitMessage.mockRestore()
|
||||
}
|
||||
}
|
||||
|
||||
test('labels an explicit gRPC cancellation', async () => {
|
||||
expect(await exerciseInterruption('cancel')).toBe('grpc_cancel')
|
||||
})
|
||||
|
||||
test('labels a gRPC stream ending while a query is active', async () => {
|
||||
expect(await exerciseInterruption('end')).toBe('grpc_stream_end')
|
||||
})
|
||||
+2
-2
@@ -289,7 +289,7 @@ export class GrpcServer {
|
||||
} else if (clientMessage.cancel) {
|
||||
interrupted = true
|
||||
if (engine) {
|
||||
engine.interrupt()
|
||||
engine.interrupt('grpc_cancel')
|
||||
}
|
||||
call.end()
|
||||
}
|
||||
@@ -312,7 +312,7 @@ export class GrpcServer {
|
||||
resolve('no')
|
||||
}
|
||||
if (engine) {
|
||||
engine.interrupt()
|
||||
engine.interrupt('grpc_stream_end')
|
||||
}
|
||||
engine = null
|
||||
pendingRequests.clear()
|
||||
|
||||
@@ -22,6 +22,7 @@ import type {
|
||||
} from '../../types/permissions.js'
|
||||
import { setClassifierApproval } from '../../utils/classifierApprovals.js'
|
||||
import { logForDebugging } from '../../utils/debug.js'
|
||||
import { requestAbort } from '../../utils/interruptionTrace.js'
|
||||
import { executePermissionRequestHooks } from '../../utils/hooks.js'
|
||||
import {
|
||||
REJECT_MESSAGE,
|
||||
@@ -210,6 +211,7 @@ function createPermissionContext(
|
||||
feedback?: string,
|
||||
isAbort?: boolean,
|
||||
contentBlocks?: ContentBlockParam[],
|
||||
abortTrace?: { source: string; causalEventId?: string },
|
||||
): PermissionDecision {
|
||||
const sub = !!toolUseContext.agentId
|
||||
const baseMessage = feedback
|
||||
@@ -222,7 +224,12 @@ function createPermissionContext(
|
||||
logForDebugging(
|
||||
`Aborting: tool=${tool.name} isAbort=${isAbort} hasFeedback=${!!feedback} isSubagent=${sub}`,
|
||||
)
|
||||
toolUseContext.abortController.abort('interrupt')
|
||||
requestAbort(toolUseContext.abortController, 'interrupt', {
|
||||
source: abortTrace?.source ?? 'permission_abort',
|
||||
causalEventId: abortTrace?.causalEventId,
|
||||
subsystem: 'tool_permission',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
return { behavior: 'ask', message, contentBlocks }
|
||||
},
|
||||
@@ -367,7 +374,11 @@ function createPermissionContext(
|
||||
logForDebugging(
|
||||
`Hook interrupt: tool=${tool.name} hookMessage=${decision.message}`,
|
||||
)
|
||||
toolUseContext.abortController.abort('interrupt')
|
||||
requestAbort(toolUseContext.abortController, 'interrupt', {
|
||||
source: 'permission_hook',
|
||||
subsystem: 'tool_permission',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
return this.buildDeny(
|
||||
decision.message || 'Permission denied by hook',
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
import { describe, expect, test, vi } from 'vitest'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
requestAbort,
|
||||
} from '../../../utils/interruptionTrace.js'
|
||||
import {
|
||||
handleInteractivePermission,
|
||||
type InteractivePermissionParams,
|
||||
@@ -9,7 +15,7 @@ import {
|
||||
// bypasses resolveOnce fails here instead of silently stranding the watchdog.
|
||||
|
||||
type QueueItem = {
|
||||
onAbort: () => void
|
||||
onAbort: (source?: string, causalEventId?: string) => void
|
||||
onAllow: (
|
||||
updatedInput: Record<string, unknown>,
|
||||
permissionUpdates: unknown[],
|
||||
@@ -118,10 +124,19 @@ describe('handleInteractivePermission watchdog suspension', () => {
|
||||
})
|
||||
|
||||
test('resumes exactly once on abort', () => {
|
||||
const { getQueueItem, resume, resolve } = setup()
|
||||
getQueueItem().onAbort()
|
||||
const { ctx, getQueueItem, resume, resolve } = setup()
|
||||
getQueueItem().onAbort('cancel_keybinding', 'input-event-1')
|
||||
expect(resume).toHaveBeenCalledTimes(1)
|
||||
expect(resolve).toHaveBeenCalledTimes(1)
|
||||
expect(ctx.cancelAndAbort).toHaveBeenCalledWith(
|
||||
undefined,
|
||||
true,
|
||||
undefined,
|
||||
{
|
||||
source: 'cancel_keybinding',
|
||||
causalEventId: 'input-event-1',
|
||||
},
|
||||
)
|
||||
})
|
||||
|
||||
test('resumes only once when two resolution paths race', async () => {
|
||||
@@ -166,6 +181,55 @@ describe('handleInteractivePermission watchdog suspension', () => {
|
||||
expect(ctx.removeFromQueue).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
test('preserves the originating trace when an external abort closes the dialog', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const { ctx, abortController } = setup()
|
||||
|
||||
try {
|
||||
requestAbort(abortController, undefined, {
|
||||
source: 'cancel_keybinding',
|
||||
subsystem: 'query_engine',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const originatingAbort = trace.find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.source === 'cancel_keybinding',
|
||||
)
|
||||
const permissionResolution = trace.find(
|
||||
entry => entry.event === 'permission.abort_resolved',
|
||||
)
|
||||
expect(typeof originatingAbort?.eventId).toBe('string')
|
||||
expect(permissionResolution).toMatchObject({
|
||||
source: 'cancel_keybinding',
|
||||
subsystem: 'tool_permission',
|
||||
causalEventId: originatingAbort!.eventId,
|
||||
outcome: 'denied',
|
||||
})
|
||||
expect(ctx.cancelAndAbort).toHaveBeenCalledWith(
|
||||
undefined,
|
||||
true,
|
||||
undefined,
|
||||
{
|
||||
source: 'cancel_keybinding',
|
||||
causalEventId: originatingAbort!.eventId,
|
||||
},
|
||||
)
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('resolves and resumes immediately if already aborted when shown', () => {
|
||||
const { ctx, resume, resolve } = setup({ preAbort: true })
|
||||
expect(resume).toHaveBeenCalledTimes(1)
|
||||
|
||||
@@ -27,6 +27,10 @@ import { errorMessage } from '../../../utils/errors.js'
|
||||
import type { PermissionDecision } from '../../../utils/permissions/PermissionResult.js'
|
||||
import type { PermissionUpdate } from '../../../utils/permissions/PermissionUpdateSchema.js'
|
||||
import { hasPermissionsToUseTool } from '../../../utils/permissions/permissions.js'
|
||||
import {
|
||||
getInterruptionSignalAbortTrace,
|
||||
tracePermissionAbortResolution,
|
||||
} from '../../../utils/interruptionTrace.js'
|
||||
import type { PermissionContext } from '../PermissionContext.js'
|
||||
import { createResolveOnce } from '../PermissionContext.js'
|
||||
|
||||
@@ -127,12 +131,23 @@ function handleInteractivePermission(
|
||||
const abortSignal = ctx.toolUseContext.abortController.signal
|
||||
const onExternalAbort = () => {
|
||||
if (!claim()) return
|
||||
const abortTrace = getInterruptionSignalAbortTrace(abortSignal)
|
||||
tracePermissionAbortResolution(
|
||||
abortTrace.source,
|
||||
abortTrace.causalEventId,
|
||||
'tool_permission',
|
||||
)
|
||||
if (bridgeCallbacks && bridgeRequestId) {
|
||||
bridgeCallbacks.cancelRequest(bridgeRequestId)
|
||||
}
|
||||
channelUnsubscribe?.()
|
||||
ctx.removeFromQueue()
|
||||
resolveOnce(ctx.cancelAndAbort(undefined, true))
|
||||
resolveOnce(
|
||||
ctx.cancelAndAbort(undefined, true, undefined, {
|
||||
source: abortTrace.source ?? 'permission_abort',
|
||||
causalEventId: abortTrace.causalEventId,
|
||||
}),
|
||||
)
|
||||
}
|
||||
if (abortSignal.aborted) {
|
||||
// Already aborted: cancel and stop setup so we never enqueue a stale prompt.
|
||||
@@ -190,7 +205,7 @@ function handleInteractivePermission(
|
||||
ctx.removeFromQueue()
|
||||
}
|
||||
},
|
||||
onAbort() {
|
||||
onAbort(source, causalEventId) {
|
||||
if (!claim()) return
|
||||
if (bridgeCallbacks && bridgeRequestId) {
|
||||
bridgeCallbacks.sendResponse(bridgeRequestId, {
|
||||
@@ -205,7 +220,12 @@ function handleInteractivePermission(
|
||||
{ decision: 'reject', source: { type: 'user_abort' } },
|
||||
{ permissionPromptStartTimeMs },
|
||||
)
|
||||
resolveOnce(ctx.cancelAndAbort(undefined, true))
|
||||
resolveOnce(
|
||||
ctx.cancelAndAbort(undefined, true, undefined, {
|
||||
source: source ?? 'permission_dialog',
|
||||
causalEventId,
|
||||
}),
|
||||
)
|
||||
},
|
||||
async onAllow(
|
||||
updatedInput,
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
import { PassThrough } from 'node:stream'
|
||||
import { afterEach, beforeEach, expect, test } from 'bun:test'
|
||||
import React, { useEffect } from 'react'
|
||||
import { createRoot } from '../ink.js'
|
||||
import { KeyboardEvent } from '../ink/events/keyboard-event.js'
|
||||
import { AppStateProvider, type AppState } from '../state/AppState.js'
|
||||
import { getDefaultAppState } from '../state/AppStateStore.js'
|
||||
import type { InProcessTeammateTaskState } from '../tasks/InProcessTeammateTask/types.js'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../test/sharedMutationLock.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
} from '../utils/interruptionTrace.js'
|
||||
import { useBackgroundTaskNavigation } from './useBackgroundTaskNavigation.js'
|
||||
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock(
|
||||
'hooks/useBackgroundTaskNavigation.interruptionTrace.test.tsx',
|
||||
)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
try {
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
} finally {
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
function Harness({
|
||||
onReady,
|
||||
}: {
|
||||
onReady: (handler: (event: KeyboardEvent) => void) => void
|
||||
}): React.ReactNode {
|
||||
const { handleKeyDown } = useBackgroundTaskNavigation()
|
||||
useEffect(() => onReady(handleKeyDown), [handleKeyDown, onReady])
|
||||
return null
|
||||
}
|
||||
|
||||
function createTeammateTask(): {
|
||||
task: InProcessTeammateTaskState
|
||||
currentWorkAbortController: AbortController
|
||||
lifecycleAbortController: AbortController
|
||||
} {
|
||||
const currentWorkAbortController = new AbortController()
|
||||
const lifecycleAbortController = new AbortController()
|
||||
return {
|
||||
currentWorkAbortController,
|
||||
lifecycleAbortController,
|
||||
task: {
|
||||
id: 'teammate-task-1',
|
||||
type: 'in_process_teammate',
|
||||
status: 'running',
|
||||
description: 'test teammate',
|
||||
startTime: Date.now(),
|
||||
outputFile: '/tmp/test-teammate-output',
|
||||
outputOffset: 0,
|
||||
notified: false,
|
||||
identity: {
|
||||
agentId: 'researcher@test-team',
|
||||
agentName: 'researcher',
|
||||
teamName: '',
|
||||
planModeRequired: false,
|
||||
parentSessionId: 'parent-session',
|
||||
},
|
||||
prompt: 'test',
|
||||
abortController: lifecycleAbortController,
|
||||
currentWorkAbortController,
|
||||
awaitingPlanApproval: false,
|
||||
permissionMode: 'default',
|
||||
isIdle: false,
|
||||
shutdownRequested: false,
|
||||
pendingUserMessages: [],
|
||||
lastReportedToolCount: 0,
|
||||
lastReportedTokenCount: 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function renderNavigation(initialState: AppState): Promise<{
|
||||
invoke: (key: 'escape' | 'k') => void
|
||||
cleanup: () => Promise<void>
|
||||
}> {
|
||||
const stdout = new PassThrough()
|
||||
const stdin = new PassThrough() as PassThrough & {
|
||||
isTTY: boolean
|
||||
setRawMode: (mode: boolean) => void
|
||||
ref: () => void
|
||||
unref: () => void
|
||||
}
|
||||
stdin.isTTY = true
|
||||
stdin.setRawMode = () => {}
|
||||
stdin.ref = () => {}
|
||||
stdin.unref = () => {}
|
||||
;(stdout as unknown as { columns: number }).columns = 120
|
||||
const root = await createRoot({
|
||||
stdout: stdout as unknown as NodeJS.WriteStream,
|
||||
stdin: stdin as unknown as NodeJS.ReadStream,
|
||||
patchConsole: false,
|
||||
})
|
||||
let handler: ((event: KeyboardEvent) => void) | undefined
|
||||
try {
|
||||
root.render(
|
||||
<AppStateProvider initialState={initialState}>
|
||||
<Harness
|
||||
onReady={value => {
|
||||
handler = value
|
||||
}}
|
||||
/>
|
||||
</AppStateProvider>,
|
||||
)
|
||||
for (let attempts = 0; attempts < 100 && !handler; attempts++) {
|
||||
await Bun.sleep(10)
|
||||
}
|
||||
expect(handler).toBeDefined()
|
||||
return {
|
||||
invoke(key) {
|
||||
handler!(
|
||||
new KeyboardEvent({
|
||||
kind: 'key',
|
||||
name: key,
|
||||
sequence: key === 'escape' ? '\u001b' : key,
|
||||
raw: key === 'escape' ? '\u001b' : key,
|
||||
ctrl: false,
|
||||
shift: false,
|
||||
meta: false,
|
||||
option: false,
|
||||
super: false,
|
||||
fn: false,
|
||||
isPasted: false,
|
||||
}),
|
||||
)
|
||||
},
|
||||
async cleanup() {
|
||||
root.unmount()
|
||||
await Bun.sleep(30)
|
||||
stdin.end()
|
||||
stdout.end()
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
root.unmount()
|
||||
await Bun.sleep(30)
|
||||
stdin.end()
|
||||
stdout.end()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
test('records Escape causality before aborting the current teammate turn', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const { task, currentWorkAbortController } = createTeammateTask()
|
||||
const rendered = await renderNavigation({
|
||||
...getDefaultAppState(),
|
||||
tasks: { [task.id]: task },
|
||||
viewingAgentTaskId: task.id,
|
||||
viewSelectionMode: 'viewing-agent',
|
||||
})
|
||||
|
||||
try {
|
||||
rendered.invoke('escape')
|
||||
const entries = __getInterruptionTraceSnapshotForTests()
|
||||
const input = entries.find(entry => entry.event === 'input.teammate_escape')
|
||||
const requested = entries.find(entry => entry.event === 'abort.requested')
|
||||
expect(currentWorkAbortController.signal.aborted).toBe(true)
|
||||
expect(currentWorkAbortController.signal.reason).toBeInstanceOf(
|
||||
DOMException,
|
||||
)
|
||||
expect(
|
||||
(currentWorkAbortController.signal.reason as DOMException).name,
|
||||
).toBe('AbortError')
|
||||
expect(input).toBeDefined()
|
||||
expect(requested).toMatchObject({
|
||||
source: 'teammate_escape',
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-turn',
|
||||
subagentId: 'researcher@test-team',
|
||||
causalEventId: input?.eventId,
|
||||
})
|
||||
} finally {
|
||||
await rendered.cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
test('records kill-key causality before aborting the teammate lifecycle', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const { task, lifecycleAbortController } = createTeammateTask()
|
||||
const rendered = await renderNavigation({
|
||||
...getDefaultAppState(),
|
||||
tasks: { [task.id]: task },
|
||||
expandedView: 'teammates',
|
||||
selectedIPAgentIndex: 0,
|
||||
viewSelectionMode: 'selecting-agent',
|
||||
})
|
||||
|
||||
try {
|
||||
rendered.invoke('k')
|
||||
const entries = __getInterruptionTraceSnapshotForTests()
|
||||
const input = entries.find(entry => entry.event === 'input.teammate_kill')
|
||||
const requested = entries.find(entry => entry.event === 'abort.requested')
|
||||
expect(lifecycleAbortController.signal.aborted).toBe(true)
|
||||
expect(input).toBeDefined()
|
||||
expect(requested).toMatchObject({
|
||||
source: 'teammate_kill',
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: 'researcher@test-team',
|
||||
causalEventId: input?.eventId,
|
||||
})
|
||||
} finally {
|
||||
await rendered.cleanup()
|
||||
}
|
||||
})
|
||||
@@ -11,15 +11,17 @@ import {
|
||||
enterTeammateView,
|
||||
exitTeammateView,
|
||||
} from '../state/teammateViewHelpers.js'
|
||||
import {
|
||||
getRunningTeammatesSorted,
|
||||
InProcessTeammateTask,
|
||||
} from '../tasks/InProcessTeammateTask/InProcessTeammateTask.js'
|
||||
import { getRunningTeammatesSorted } from '../tasks/InProcessTeammateTask/InProcessTeammateTask.js'
|
||||
import {
|
||||
type InProcessTeammateTaskState,
|
||||
isInProcessTeammateTask,
|
||||
} from '../tasks/InProcessTeammateTask/types.js'
|
||||
import { isBackgroundTask } from '../tasks/types.js'
|
||||
import {
|
||||
requestAbort,
|
||||
traceInterruptionEvent,
|
||||
} from '../utils/interruptionTrace.js'
|
||||
import { killInProcessTeammate } from '../utils/swarm/spawnInProcess.js'
|
||||
|
||||
// Step teammate selection by delta, wrapping across leader(-1)..teammates(0..n-1)..hide(n).
|
||||
// First step from a collapsed tree expands it and parks on leader.
|
||||
@@ -155,7 +157,23 @@ export function useBackgroundTaskNavigation(options?: {
|
||||
const task = tasks[taskId]
|
||||
if (isInProcessTeammateTask(task) && task.status === 'running') {
|
||||
// Abort currentWorkAbortController (stops current turn) NOT abortController (kills teammate)
|
||||
task.currentWorkAbortController?.abort()
|
||||
const causalEventId = traceInterruptionEvent(
|
||||
'input.teammate_escape',
|
||||
{
|
||||
source: 'teammate_escape',
|
||||
subsystem: 'in_process_teammate',
|
||||
subagentId: task.identity.agentId,
|
||||
},
|
||||
)
|
||||
if (task.currentWorkAbortController) {
|
||||
requestAbort(task.currentWorkAbortController, undefined, {
|
||||
source: 'teammate_escape',
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-turn',
|
||||
subagentId: task.identity.agentId,
|
||||
causalEventId,
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -233,7 +251,15 @@ export function useBackgroundTaskNavigation(options?: {
|
||||
e.preventDefault()
|
||||
const selected = getSelectedTeammate()
|
||||
if (selected && selected.task.status === 'running') {
|
||||
void InProcessTeammateTask.kill(selected.taskId, setAppState)
|
||||
const causalEventId = traceInterruptionEvent('input.teammate_kill', {
|
||||
source: 'teammate_kill',
|
||||
subsystem: 'in_process_teammate',
|
||||
subagentId: selected.task.identity.agentId,
|
||||
})
|
||||
killInProcessTeammate(selected.taskId, setAppState, {
|
||||
source: 'teammate_kill',
|
||||
causalEventId,
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
import { PassThrough } from 'node:stream'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
mock,
|
||||
spyOn,
|
||||
test,
|
||||
} from 'bun:test'
|
||||
import React from 'react'
|
||||
import * as analytics from '../services/analytics/index.js'
|
||||
import { createRoot } from '../ink.js'
|
||||
import { KeybindingProvider } from '../keybindings/KeybindingContext.js'
|
||||
import type {
|
||||
KeybindingContextName,
|
||||
ParsedKeystroke,
|
||||
} from '../keybindings/types.js'
|
||||
import { AppStateProvider } from '../state/AppState.js'
|
||||
import {
|
||||
getDefaultAppState,
|
||||
type AppState,
|
||||
} from '../state/AppStateStore.js'
|
||||
import { CancelRequestHandler } from './useCancelRequest.js'
|
||||
import type { LocalAgentTaskState } from '../tasks/LocalAgentTask/LocalAgentTask.js'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../test/sharedMutationLock.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
} from '../utils/interruptionTrace.js'
|
||||
|
||||
type HandlerRegistration = {
|
||||
action: string
|
||||
context: KeybindingContextName
|
||||
handler: () => void
|
||||
}
|
||||
|
||||
function TestKeybindingProvider({
|
||||
children,
|
||||
registry,
|
||||
}: {
|
||||
children: React.ReactNode
|
||||
registry: React.RefObject<Map<string, Set<HandlerRegistration>>>
|
||||
}): React.ReactNode {
|
||||
const pendingChordRef = React.useRef<ParsedKeystroke[] | null>(null)
|
||||
const [pendingChord, setPendingChord] = React.useState<
|
||||
ParsedKeystroke[] | null
|
||||
>(null)
|
||||
const [activeContexts] = React.useState(
|
||||
() => new Set<KeybindingContextName>(['Chat', 'Global']),
|
||||
)
|
||||
return (
|
||||
<KeybindingProvider
|
||||
bindings={[]}
|
||||
pendingChordRef={pendingChordRef}
|
||||
pendingChord={pendingChord}
|
||||
setPendingChord={setPendingChord}
|
||||
activeContexts={activeContexts}
|
||||
registerActiveContext={context => activeContexts.add(context)}
|
||||
unregisterActiveContext={context => activeContexts.delete(context)}
|
||||
handlerRegistryRef={registry}
|
||||
>
|
||||
{children}
|
||||
</KeybindingProvider>
|
||||
)
|
||||
}
|
||||
|
||||
function createTestStreams() {
|
||||
const stdout = new PassThrough()
|
||||
const stdin = new PassThrough() as PassThrough & {
|
||||
isTTY: boolean
|
||||
setRawMode: (mode: boolean) => void
|
||||
ref: () => void
|
||||
unref: () => void
|
||||
}
|
||||
stdin.isTTY = true
|
||||
stdin.setRawMode = () => {}
|
||||
stdin.ref = () => {}
|
||||
stdin.unref = () => {}
|
||||
;(stdout as unknown as { columns: number }).columns = 120
|
||||
return { stdout, stdin }
|
||||
}
|
||||
|
||||
async function waitFor(
|
||||
predicate: () => boolean,
|
||||
label: string,
|
||||
timeoutMs = 2_000,
|
||||
): Promise<void> {
|
||||
const startedAt = Date.now()
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
if (predicate()) return
|
||||
await Bun.sleep(10)
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${label}`)
|
||||
}
|
||||
|
||||
async function renderCancelHandler(
|
||||
initialState: AppState,
|
||||
options: { failAfterRender?: boolean; onCleanup?: () => void } = {},
|
||||
) {
|
||||
const { stdout, stdin } = createTestStreams()
|
||||
const root = await createRoot({
|
||||
stdout: stdout as unknown as NodeJS.WriteStream,
|
||||
stdin: stdin as unknown as NodeJS.ReadStream,
|
||||
patchConsole: false,
|
||||
})
|
||||
const registry = {
|
||||
current: new Map<string, Set<HandlerRegistration>>(),
|
||||
}
|
||||
const onCancel = mock((_source: string, _causalEventId?: string) => {})
|
||||
let latestState = initialState
|
||||
const cleanup = async () => {
|
||||
root.unmount()
|
||||
await Bun.sleep(30)
|
||||
stdin.end()
|
||||
stdout.end()
|
||||
options.onCleanup?.()
|
||||
}
|
||||
try {
|
||||
root.render(
|
||||
<AppStateProvider
|
||||
initialState={initialState}
|
||||
onChangeAppState={({ newState }) => {
|
||||
latestState = newState
|
||||
}}
|
||||
>
|
||||
<TestKeybindingProvider registry={registry}>
|
||||
<CancelRequestHandler
|
||||
setToolUseConfirmQueue={() => {}}
|
||||
onCancel={onCancel}
|
||||
onAgentsKilled={() => {}}
|
||||
isMessageSelectorVisible={false}
|
||||
screen="prompt"
|
||||
abortSignal={new AbortController().signal}
|
||||
/>
|
||||
</TestKeybindingProvider>
|
||||
</AppStateProvider>,
|
||||
)
|
||||
if (options.failAfterRender) throw new Error('synthetic setup failure')
|
||||
await waitFor(
|
||||
() =>
|
||||
registry.current.has('app:interrupt') &&
|
||||
(initialState.viewSelectionMode === 'viewing-agent' ||
|
||||
registry.current.has('chat:cancel')),
|
||||
'cancel keybinding registration',
|
||||
)
|
||||
} catch (error) {
|
||||
await cleanup()
|
||||
throw error
|
||||
}
|
||||
return {
|
||||
onCancel,
|
||||
invoke(action: 'chat:cancel' | 'app:interrupt') {
|
||||
const registration = registry.current.get(action)?.values().next().value
|
||||
if (!registration) throw new Error(`Missing ${action} handler`)
|
||||
registration.handler()
|
||||
},
|
||||
getState: () => latestState,
|
||||
cleanup,
|
||||
}
|
||||
}
|
||||
|
||||
const originalInterruptionTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
let hasSharedMutationLock = false
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock('hooks/useCancelRequest.test.tsx')
|
||||
hasSharedMutationLock = true
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
mock.restore()
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalInterruptionTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalInterruptionTrace
|
||||
}
|
||||
} finally {
|
||||
if (hasSharedMutationLock) {
|
||||
releaseSharedMutationLock()
|
||||
hasSharedMutationLock = false
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
describe('CancelRequestHandler interruption sources', () => {
|
||||
test('cleans up the Ink root and streams when setup fails', async () => {
|
||||
let cleanupCount = 0
|
||||
|
||||
await expect(
|
||||
renderCancelHandler(getDefaultAppState(), {
|
||||
failAfterRender: true,
|
||||
onCleanup: () => {
|
||||
cleanupCount++
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow('synthetic setup failure')
|
||||
expect(cleanupCount).toBe(1)
|
||||
})
|
||||
|
||||
test('chat:cancel preserves escape analytics and passes the precise cancel source', async () => {
|
||||
const logEvent = spyOn(analytics, 'logEvent').mockImplementation(() => {})
|
||||
const rendered = await renderCancelHandler(getDefaultAppState())
|
||||
try {
|
||||
rendered.invoke('chat:cancel')
|
||||
|
||||
expect(rendered.onCancel).toHaveBeenCalledWith(
|
||||
'cancel_keybinding',
|
||||
undefined,
|
||||
)
|
||||
expect(logEvent).toHaveBeenCalledWith(
|
||||
'tengu_cancel',
|
||||
expect.objectContaining({ source: 'escape' }),
|
||||
)
|
||||
} finally {
|
||||
await rendered.cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
test('app:interrupt preserves escape analytics while passing ctrl_c', async () => {
|
||||
const logEvent = spyOn(analytics, 'logEvent').mockImplementation(() => {})
|
||||
const rendered = await renderCancelHandler({
|
||||
...getDefaultAppState(),
|
||||
viewSelectionMode: 'viewing-agent',
|
||||
})
|
||||
try {
|
||||
rendered.invoke('app:interrupt')
|
||||
|
||||
expect(rendered.onCancel).toHaveBeenCalledWith('ctrl_c', undefined)
|
||||
expect(rendered.getState().viewSelectionMode).toBe('none')
|
||||
expect(logEvent).toHaveBeenCalledWith(
|
||||
'tengu_cancel',
|
||||
expect.objectContaining({ source: 'escape' }),
|
||||
)
|
||||
} finally {
|
||||
await rendered.cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
test('app:interrupt links background-agent aborts to the Ctrl-C input', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const agentAbortController = new AbortController()
|
||||
const agentTask: LocalAgentTaskState = {
|
||||
id: 'agent-1',
|
||||
type: 'local_agent',
|
||||
status: 'running',
|
||||
description: 'trace test agent',
|
||||
startTime: Date.now(),
|
||||
outputFile: join(tmpdir(), 'openclaude-trace-test-agent.output'),
|
||||
outputOffset: 0,
|
||||
notified: false,
|
||||
agentId: 'agent-1',
|
||||
prompt: 'test',
|
||||
agentType: 'general-purpose',
|
||||
abortController: agentAbortController,
|
||||
retrieved: false,
|
||||
lastReportedToolCount: 0,
|
||||
lastReportedTokenCount: 0,
|
||||
isBackgrounded: true,
|
||||
pendingMessages: [],
|
||||
retain: false,
|
||||
diskLoaded: false,
|
||||
}
|
||||
const rendered = await renderCancelHandler({
|
||||
...getDefaultAppState(),
|
||||
viewSelectionMode: 'viewing-agent',
|
||||
viewingAgentTaskId: 'agent-1',
|
||||
tasks: { 'agent-1': agentTask },
|
||||
})
|
||||
try {
|
||||
rendered.invoke('app:interrupt')
|
||||
await waitFor(
|
||||
() => agentAbortController.signal.aborted,
|
||||
'background-agent abort',
|
||||
)
|
||||
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const input = trace.find(entry => entry.event === 'input.ctrl_c')
|
||||
const agentAbort = trace.find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.controllerRole === 'background-agent',
|
||||
)
|
||||
expect(input).toBeDefined()
|
||||
expect(agentAbort).toMatchObject({
|
||||
source: 'ctrl_c',
|
||||
subagentId: 'agent-1',
|
||||
causalEventId: input!.eventId,
|
||||
})
|
||||
} finally {
|
||||
await rendered.cleanup()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -33,15 +33,18 @@ import {
|
||||
hasCommandsInQueue,
|
||||
} from '../utils/messageQueueManager.js'
|
||||
import { emitTaskTerminatedSdk } from '../utils/sdkEventQueue.js'
|
||||
import { traceInterruptionEvent } from '../utils/interruptionTrace.js'
|
||||
|
||||
/** Time window in ms during which a second press kills all background agents. */
|
||||
const KILL_AGENTS_CONFIRM_WINDOW_MS = 3000
|
||||
|
||||
export type CancelRequestSource = 'cancel_keybinding' | 'ctrl_c'
|
||||
|
||||
type CancelRequestHandlerProps = {
|
||||
setToolUseConfirmQueue: (
|
||||
f: (toolUseConfirmQueue: ToolUseConfirm[]) => ToolUseConfirm[],
|
||||
) => void
|
||||
onCancel: () => void
|
||||
onCancel: (source: CancelRequestSource, causalEventId?: string) => void
|
||||
onAgentsKilled: () => void
|
||||
isMessageSelectorVisible: boolean
|
||||
screen: Screen
|
||||
@@ -84,7 +87,10 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null {
|
||||
const lastKillAgentsPressRef = useRef<number>(0)
|
||||
const viewSelectionMode = useAppState(s => s.viewSelectionMode)
|
||||
|
||||
const handleCancel = useCallback(() => {
|
||||
const handleCancel = useCallback((
|
||||
source: CancelRequestSource,
|
||||
causalEventId?: string,
|
||||
) => {
|
||||
const cancelProps = {
|
||||
source:
|
||||
'escape' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
@@ -97,7 +103,7 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null {
|
||||
if (abortSignal !== undefined && !abortSignal.aborted) {
|
||||
logEvent('tengu_cancel', cancelProps)
|
||||
setToolUseConfirmQueue(() => [])
|
||||
onCancel()
|
||||
onCancel(source, causalEventId)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -112,7 +118,7 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null {
|
||||
// Fallback: nothing to cancel or pop (shouldn't reach here if isActive is correct)
|
||||
logEvent('tengu_cancel', cancelProps)
|
||||
setToolUseConfirmQueue(() => [])
|
||||
onCancel()
|
||||
onCancel(source, causalEventId)
|
||||
}, [
|
||||
abortSignal,
|
||||
popCommandFromQueue,
|
||||
@@ -161,7 +167,18 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null {
|
||||
isContextActive &&
|
||||
(canCancelRunningTask || hasQueuedCommands || isViewingTeammate)
|
||||
|
||||
useKeybinding('chat:cancel', handleCancel, {
|
||||
const handleCancelKeybinding = useCallback(
|
||||
() => {
|
||||
const causalEventId = traceInterruptionEvent('input.cancel_keybinding', {
|
||||
source: 'cancel_keybinding',
|
||||
subsystem: 'cancel_request',
|
||||
})
|
||||
handleCancel('cancel_keybinding', causalEventId)
|
||||
},
|
||||
[handleCancel],
|
||||
)
|
||||
|
||||
useKeybinding('chat:cancel', handleCancelKeybinding, {
|
||||
context: 'Chat',
|
||||
isActive: isEscapeActive,
|
||||
})
|
||||
@@ -169,13 +186,14 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null {
|
||||
// Shared kill path: stop all agents, suppress per-agent notifications,
|
||||
// emit SDK events, enqueue a single aggregate model-facing notification.
|
||||
// Returns true if anything was killed.
|
||||
const killAllAgentsAndNotify = useCallback((): boolean => {
|
||||
const killAllAgentsAndNotify = useCallback(
|
||||
(source: string, causalEventId?: string): boolean => {
|
||||
const tasks = store.getState().tasks
|
||||
const running = Object.entries(tasks).filter(
|
||||
([, t]) => t.type === 'local_agent' && t.status === 'running',
|
||||
)
|
||||
if (running.length === 0) return false
|
||||
killAllRunningAgentTasks(tasks, setAppState)
|
||||
killAllRunningAgentTasks(tasks, setAppState, { source, causalEventId })
|
||||
const descriptions: string[] = []
|
||||
for (const [taskId, task] of running) {
|
||||
markAgentsNotified(taskId, setAppState)
|
||||
@@ -192,18 +210,25 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null {
|
||||
enqueuePendingNotification({ value: summary, mode: 'task-notification' })
|
||||
onAgentsKilled()
|
||||
return true
|
||||
}, [store, setAppState, onAgentsKilled])
|
||||
},
|
||||
[store, setAppState, onAgentsKilled],
|
||||
)
|
||||
|
||||
// Ctrl+C (app:interrupt). Scoped to teammate-view: killing agents from the
|
||||
// main prompt stays a deliberate gesture (chat:killAgents), not a
|
||||
// side-effect of cancelling a turn.
|
||||
const handleInterrupt = useCallback(() => {
|
||||
const causalEventId = traceInterruptionEvent('input.ctrl_c', {
|
||||
source: 'ctrl_c',
|
||||
subsystem: 'cancel_request',
|
||||
phase: isViewingTeammate ? 'teammate_view' : 'main_view',
|
||||
})
|
||||
if (isViewingTeammate) {
|
||||
killAllAgentsAndNotify()
|
||||
killAllAgentsAndNotify('ctrl_c', causalEventId)
|
||||
exitTeammateView(setAppState)
|
||||
}
|
||||
if (canCancelRunningTask || hasQueuedCommands) {
|
||||
handleCancel()
|
||||
handleCancel('ctrl_c', causalEventId)
|
||||
}
|
||||
}, [
|
||||
isViewingTeammate,
|
||||
@@ -247,7 +272,11 @@ export function CancelRequestHandler(props: CancelRequestHandlerProps): null {
|
||||
'kill_agents' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
})
|
||||
clearCommandQueue()
|
||||
killAllAgentsAndNotify()
|
||||
const causalEventId = traceInterruptionEvent('input.kill_agents', {
|
||||
source: 'kill_agents',
|
||||
subsystem: 'cancel_request',
|
||||
})
|
||||
killAllAgentsAndNotify('kill_agents', causalEventId)
|
||||
return
|
||||
}
|
||||
// First press -- show confirmation hint in status bar
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
handlePlanApprovalResponse,
|
||||
} from '../utils/inProcessTeammateHelpers.js'
|
||||
import { createAssistantMessage } from '../utils/messages.js'
|
||||
import { tracePermissionAbortResolution } from '../utils/interruptionTrace.js'
|
||||
import { requestPermissionModeChange } from '../utils/permissions/permissionModeChange.js'
|
||||
import {
|
||||
applyPermissionModeChange,
|
||||
@@ -313,7 +314,12 @@ export function useInboxPoller({
|
||||
onUserInteraction() {
|
||||
// No-op for tmux workers (no classifier auto-approval)
|
||||
},
|
||||
onAbort() {
|
||||
onAbort(source, causalEventId) {
|
||||
tracePermissionAbortResolution(
|
||||
source,
|
||||
causalEventId,
|
||||
'tmux_permission_bridge',
|
||||
)
|
||||
void sendPermissionResponseViaMailbox(
|
||||
parsed.agent_id,
|
||||
{ decision: 'rejected', resolvedBy: 'leader' },
|
||||
|
||||
@@ -20,6 +20,7 @@ import type { Message } from '../types/message.js';
|
||||
import { getCwd } from '../utils/cwd.js';
|
||||
import { logForDebugging } from '../utils/debug.js';
|
||||
import { errorMessage } from '../utils/errors.js';
|
||||
import { requestBridgeInterrupt } from '../utils/replInterruption.js';
|
||||
import { enqueue } from '../utils/messageQueueManager.js';
|
||||
import { buildSystemInitMessage } from '../utils/messages/systemInit.js';
|
||||
import { createBridgeStatusMessage, createSystemMessage } from '../utils/messages.js';
|
||||
@@ -393,7 +394,7 @@ export function useReplBridge(messages: Message[], setMessages: (action: React.S
|
||||
onInboundMessage: handleInboundMessage,
|
||||
onPermissionResponse: handlePermissionResponse,
|
||||
onInterrupt() {
|
||||
abortControllerRef.current?.abort('interrupt');
|
||||
requestBridgeInterrupt(abortControllerRef);
|
||||
},
|
||||
onSetModel(model) {
|
||||
const resolved = model === 'default' ? null : model ?? null;
|
||||
|
||||
@@ -28,6 +28,7 @@ import type { Message as MessageType } from '../types/message.js'
|
||||
import type { PermissionAskDecision } from '../types/permissions.js'
|
||||
import { logForDebugging } from '../utils/debug.js'
|
||||
import { gracefulShutdown } from '../utils/gracefulShutdown.js'
|
||||
import { tracePermissionAbortResolution } from '../utils/interruptionTrace.js'
|
||||
import type { RemoteMessageContent } from '../utils/teleport/api.js'
|
||||
|
||||
type UseSSHSessionResult = {
|
||||
@@ -121,7 +122,12 @@ export function useSSHSession({
|
||||
permissionResult,
|
||||
permissionPromptStartTimeMs: Date.now(),
|
||||
onUserInteraction() {},
|
||||
onAbort() {
|
||||
onAbort(source, causalEventId) {
|
||||
tracePermissionAbortResolution(
|
||||
source,
|
||||
causalEventId,
|
||||
'ssh_permission_bridge',
|
||||
)
|
||||
manager.respondToPermissionRequest(requestId, {
|
||||
behavior: 'deny',
|
||||
message: 'User aborted',
|
||||
|
||||
@@ -11,6 +11,16 @@ import {
|
||||
INTERRUPT_MESSAGE,
|
||||
} from './utils/messages.js'
|
||||
import { asSystemPrompt } from './utils/systemPromptType.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
requestAbort,
|
||||
} from './utils/interruptionTrace.js'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from './test/sharedMutationLock.js'
|
||||
|
||||
const DEFAULT_ABORT = Symbol('default abort')
|
||||
|
||||
@@ -55,11 +65,10 @@ function makeToolUseContext(tools: Tools = []): QueryParams['toolUseContext'] {
|
||||
}
|
||||
|
||||
function abort(controller: AbortController, reason: AbortInput): void {
|
||||
if (reason === DEFAULT_ABORT) {
|
||||
controller.abort()
|
||||
return
|
||||
}
|
||||
controller.abort(reason)
|
||||
requestAbort(controller, reason === DEFAULT_ABORT ? undefined : reason, {
|
||||
source: 'query_abort_test',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
|
||||
function makeBaseParams(
|
||||
@@ -170,6 +179,33 @@ function interruptionMessages(yielded: any[]) {
|
||||
}
|
||||
|
||||
describe('query abort classification', () => {
|
||||
test.serial('links abort classification to the winning root abort', async () => {
|
||||
await acquireSharedMutationLock('query abort classification trace')
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
try {
|
||||
await drainWithReturn(makeParams('query-timeout'))
|
||||
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const rootAbort = trace.find(entry => entry.event === 'abort.requested')
|
||||
const classified = trace.find(
|
||||
entry => entry.event === 'query.abort_classified',
|
||||
)
|
||||
expect(rootAbort).toBeDefined()
|
||||
expect(classified).toBeDefined()
|
||||
expect(typeof rootAbort!.eventId).toBe('string')
|
||||
expect(typeof classified!.causalEventId).toBe('string')
|
||||
expect(classified!.causalEventId).toBe(rootAbort!.eventId)
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
test('query timeout aborts produce timeout transcript text without user interruption', async () => {
|
||||
const { yielded, returned } = await drainWithReturn(
|
||||
makeParams('query-timeout'),
|
||||
|
||||
@@ -5,6 +5,7 @@ import type {
|
||||
} from '@anthropic-ai/sdk/resources/index.mjs'
|
||||
import type { CanUseToolFn } from './hooks/useCanUseTool.js'
|
||||
import { FallbackTriggeredError } from './services/api/withRetry.js'
|
||||
import { isMainThreadGoalSource } from './services/goal/controller.js'
|
||||
import {
|
||||
calculateTokenWarningState,
|
||||
getAutoCompactThreshold,
|
||||
@@ -55,6 +56,11 @@ import {
|
||||
normalizeAbortReason,
|
||||
shouldCreateUserInterruptionMessage,
|
||||
} from './utils/abortReasons.js'
|
||||
import {
|
||||
flushInterruptionTrace,
|
||||
getInterruptionSignalAbortEventId,
|
||||
traceInterruptionEvent,
|
||||
} from './utils/interruptionTrace.js'
|
||||
import {
|
||||
createAssistantMessage,
|
||||
createUserMessage,
|
||||
@@ -185,6 +191,27 @@ async function cleanupComputerUseAtTerminal(
|
||||
}
|
||||
}
|
||||
|
||||
function traceAbortMessageSelection(
|
||||
signal: AbortSignal,
|
||||
phase: 'streaming' | 'tools' | 'post-tools',
|
||||
): void {
|
||||
const abortReason = signal.reason
|
||||
const createsUserInterruption = shouldCreateUserInterruptionMessage(abortReason)
|
||||
const createsSystemWarning = getQueryAbortSystemMessage(abortReason) !== null
|
||||
traceInterruptionEvent('query.abort_classified', {
|
||||
subsystem: 'query',
|
||||
phase,
|
||||
reason: abortReason,
|
||||
causalEventId: getInterruptionSignalAbortEventId(signal),
|
||||
outcome: createsUserInterruption
|
||||
? 'user_interruption'
|
||||
: createsSystemWarning
|
||||
? 'system_warning'
|
||||
: 'silent',
|
||||
})
|
||||
flushInterruptionTrace('query_abort_classified')
|
||||
}
|
||||
|
||||
async function* emitAbortedStreaming(
|
||||
signal: AbortSignal,
|
||||
toolUseContext: ToolUseContext,
|
||||
@@ -194,6 +221,7 @@ async function* emitAbortedStreaming(
|
||||
> {
|
||||
await cleanupComputerUseAtTerminal(toolUseContext)
|
||||
const abortReason = signal.reason
|
||||
traceAbortMessageSelection(signal, 'streaming')
|
||||
const abortSystemMessage = getQueryAbortSystemMessage(abortReason)
|
||||
if (abortSystemMessage) {
|
||||
yield createSystemMessage(abortSystemMessage, 'warning')
|
||||
@@ -211,6 +239,7 @@ function* emitAbortedToolsAfterCleanup(
|
||||
hasSharedTurnBudget: boolean,
|
||||
): Generator<Message, Extract<Terminal, { reason: 'aborted_tools' }>> {
|
||||
const abortReason = signal.reason
|
||||
traceAbortMessageSelection(signal, 'tools')
|
||||
const abortSystemMessage = getQueryAbortSystemMessage(abortReason)
|
||||
if (abortSystemMessage) {
|
||||
yield createSystemMessage(abortSystemMessage, 'warning')
|
||||
@@ -794,6 +823,19 @@ async function* queryLoop(
|
||||
state.toolUseContext,
|
||||
)
|
||||
|
||||
const activeGoal = state.toolUseContext.getAppState().goal
|
||||
if (
|
||||
activeGoal?.status === 'active' &&
|
||||
isMainThreadGoalSource(querySource, state.toolUseContext)
|
||||
) {
|
||||
traceInterruptionEvent('goal.main_turn_started', {
|
||||
subsystem: 'goal',
|
||||
phase: 'main_query',
|
||||
querySource,
|
||||
attemptId: activeGoal.id,
|
||||
})
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
// Destructure state at the top of each iteration. toolUseContext alone
|
||||
@@ -2007,6 +2049,7 @@ async function* queryLoop(
|
||||
// Without this, tool_use blocks would lack matching tool_result blocks.
|
||||
if (toolUseContext.abortController.signal.aborted) {
|
||||
const abortReason = toolUseContext.abortController.signal.reason
|
||||
traceAbortMessageSelection(toolUseContext.abortController.signal, 'post-tools')
|
||||
if (streamingToolExecutor) {
|
||||
// Consume remaining results - executor generates synthetic tool_results for
|
||||
// aborted tools since it checks the abort signal in executeTool()
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { AssistantMessage } from '../types/message.js'
|
||||
import type { PermissionAskDecision } from '../types/permissions.js'
|
||||
import { jsonStringify } from '../utils/slowOperations.js'
|
||||
import type { ToolUseConfirm } from '../components/permissions/PermissionRequest.js'
|
||||
import { tracePermissionAbortResolution } from '../utils/interruptionTrace.js'
|
||||
|
||||
/**
|
||||
* Create a synthetic AssistantMessage for remote permission requests.
|
||||
@@ -123,7 +124,12 @@ export function createRemotePermissionQueueItem({
|
||||
onUserInteraction() {
|
||||
// No-op for remote permission prompts.
|
||||
},
|
||||
onAbort() {
|
||||
onAbort(source, causalEventId) {
|
||||
tracePermissionAbortResolution(
|
||||
source,
|
||||
causalEventId,
|
||||
'remote_permission_bridge',
|
||||
)
|
||||
respond({
|
||||
behavior: 'deny',
|
||||
message: 'User aborted',
|
||||
|
||||
+91
-27
@@ -38,7 +38,7 @@ import { asSessionId, asAgentId } from '../types/ids.js';
|
||||
import { logForDebugging } from '../utils/debug.js';
|
||||
import { QueryGuard } from '../utils/QueryGuard.js';
|
||||
import { getQueryGuardOptionsFromEnv } from '../utils/queryGuardConfig.js';
|
||||
import { QueryLifecycleOperationTracker, formatQueryLifecycleAbortSignalReason, formatQueryLifecycleLogMessage, getQueryTerminalReason, type QueryActiveOperationSnapshot, type QueryGuardTimeoutInfo, type QueryLifecycleContext, type QueryTerminalReason } from '../utils/queryLifecycle.js';
|
||||
import { QueryLifecycleOperationTracker, formatQueryLifecycleAbortSignalReason, formatQueryLifecycleLogMessage, getQueryTerminalOutcome, getQueryTerminalReason, type QueryActiveOperationSnapshot, type QueryGuardTimeoutInfo, type QueryLifecycleContext, type QueryTerminalReason } from '../utils/queryLifecycle.js';
|
||||
import { claimBackgroundTurnBudget, canRestoreDeferredMaxTurnsCap, computeDeferredMaxTurnsCapForBackgroundHandoff, createForegroundTurnBudgetHandoff, getReplMaxTurnsWarning, releaseForegroundTurnBudget, resolveReplMaxTurnsForSession, shouldShowReplMaxTurnsUnlimitedWarning, shouldContinueBackgroundAfterForegroundQuery, waitForForegroundTurnBudgetSettlement, type ForegroundTurnBudgetHandoff } from './replMaxTurns.js';
|
||||
import { createCombinedAbortSignal } from '../utils/combinedAbortSignal.js';
|
||||
import { isEnvTruthy } from '../utils/envUtils.js';
|
||||
@@ -92,7 +92,10 @@ import { CommandKeybindingHandlers } from '../hooks/useCommandKeybindings.js';
|
||||
import { KeybindingSetup } from '../keybindings/KeybindingProviderSetup.js';
|
||||
import { useShortcutDisplay } from '../keybindings/useShortcutDisplay.js';
|
||||
import { getShortcutDisplay } from '../keybindings/shortcutFormat.js';
|
||||
import { CancelRequestHandler } from '../hooks/useCancelRequest.js';
|
||||
import {
|
||||
CancelRequestHandler,
|
||||
type CancelRequestSource,
|
||||
} from '../hooks/useCancelRequest.js';
|
||||
import { useBackgroundTaskNavigation } from '../hooks/useBackgroundTaskNavigation.js';
|
||||
import { useSwarmInitialization } from '../hooks/useSwarmInitialization.js';
|
||||
import { useTeammateViewAutoExit } from '../hooks/useTeammateViewAutoExit.js';
|
||||
@@ -224,6 +227,18 @@ import { RemoteCallout } from '../components/RemoteCallout.js';
|
||||
import { getAPIProvider } from '../utils/model/providers.js';
|
||||
import { activityManager } from '../utils/activityManager.js';
|
||||
import { createAbortController } from '../utils/abortController.js';
|
||||
import {
|
||||
flushInterruptionTrace,
|
||||
getInterruptionSignalAbortEventId,
|
||||
registerInterruptionController,
|
||||
requestAbort,
|
||||
traceInterruptionEvent,
|
||||
} from '../utils/interruptionTrace.js';
|
||||
import { driveQueryEvents } from '../utils/queryEventDriver.js';
|
||||
import {
|
||||
requestBackgroundHandoffAbort,
|
||||
requestPriorityNowAbort,
|
||||
} from '../utils/replInterruption.js';
|
||||
import { MCPConnectionManager } from 'src/services/mcp/MCPConnectionManager.js';
|
||||
import { useFeedbackSurvey } from 'src/components/FeedbackSurvey/useFeedbackSurvey.js';
|
||||
import { useMemorySurvey } from 'src/components/FeedbackSurvey/useMemorySurvey.js';
|
||||
@@ -1887,7 +1902,20 @@ export function REPL({
|
||||
const activeAbortController = abortControllerRef.current;
|
||||
if (activeAbortController && !activeAbortController.signal.aborted) {
|
||||
logQueryLifecycle('abort_requested', timeout.context, formatQueryLifecycleAbortSignalReason(timeoutAbortReason));
|
||||
activeAbortController.abort(timeoutAbortReason);
|
||||
requestAbort(activeAbortController, timeoutAbortReason, {
|
||||
source: 'query_guard',
|
||||
causalEventId: timeout.causalEventId,
|
||||
subsystem: 'repl',
|
||||
phase: 'timeout',
|
||||
queryId: timeout.context.queryId,
|
||||
queryGeneration: timeout.generation,
|
||||
querySource: timeout.context.querySource,
|
||||
controllerRole: 'query-root',
|
||||
elapsedQueryMs: timeout.elapsedMs,
|
||||
activeApiCallCount: timeout.activeOperations.apiCalls.length,
|
||||
activeToolUseCount: timeout.activeOperations.toolUses.length,
|
||||
});
|
||||
flushInterruptionTrace('query_guard_timeout');
|
||||
}
|
||||
if (timeout.activeOperations.apiCalls.length > 0) {
|
||||
logForDebugging(`api.call.active_on_abort queryId=${timeout.context.queryId} generation=${timeout.generation} ${timeoutOperations}`);
|
||||
@@ -2382,7 +2410,11 @@ export function REPL({
|
||||
}, [focusedInputDialog, repinScroll]);
|
||||
// Omitted means a programmatic edit/restore cancellation, which must not arm
|
||||
// correction context because those flows rewind the conversation themselves.
|
||||
function onCancel(isUserInitiated = false) {
|
||||
function onCancel(
|
||||
isUserInitiated = false,
|
||||
cancelSource: CancelRequestSource | 'programmatic' = 'programmatic',
|
||||
causalEventId?: string,
|
||||
) {
|
||||
if (focusedInputDialog === 'elicitation') {
|
||||
// Elicitation dialog handles its own Escape, and closing it shouldn't affect any loading state.
|
||||
return;
|
||||
@@ -2436,7 +2468,7 @@ export function REPL({
|
||||
}
|
||||
if (focusedInputDialog === 'tool-permission') {
|
||||
// Tool use confirm handles the abort signal itself
|
||||
toolUseConfirmQueue[0]?.onAbort();
|
||||
toolUseConfirmQueue[0]?.onAbort(cancelSource, causalEventId);
|
||||
setToolUseConfirmQueue([]);
|
||||
} else if (focusedInputDialog === 'prompt') {
|
||||
// Reject all pending prompts and clear the queue
|
||||
@@ -2444,12 +2476,26 @@ export function REPL({
|
||||
item.reject(new Error('Prompt cancelled by user'));
|
||||
}
|
||||
setPromptQueue([]);
|
||||
abortController?.abort('user-cancel');
|
||||
if (abortController) {
|
||||
requestAbort(abortController, 'user-cancel', {
|
||||
source: cancelSource,
|
||||
causalEventId,
|
||||
subsystem: 'repl',
|
||||
controllerRole: 'query-root',
|
||||
});
|
||||
}
|
||||
} else if (activeRemote.isRemoteMode) {
|
||||
// Remote mode: send interrupt signal to CCR
|
||||
activeRemote.cancelRequest();
|
||||
} else {
|
||||
abortController?.abort('user-cancel');
|
||||
if (abortController) {
|
||||
requestAbort(abortController, 'user-cancel', {
|
||||
source: cancelSource,
|
||||
causalEventId,
|
||||
subsystem: 'repl',
|
||||
controllerRole: 'query-root',
|
||||
});
|
||||
}
|
||||
}
|
||||
if (cancelContext) {
|
||||
logQueryLifecycle('abort_acknowledged', cancelContext, formatQueryLifecycleAbortSignalReason('user-cancel'));
|
||||
@@ -2496,7 +2542,7 @@ export function REPL({
|
||||
// CancelRequestHandler props - rendered inside KeybindingSetup
|
||||
const cancelRequestProps = {
|
||||
setToolUseConfirmQueue,
|
||||
onCancel: () => onCancel(true),
|
||||
onCancel: (source, causalEventId) => onCancel(true, source, causalEventId),
|
||||
onAgentsKilled: () => setMessages(prev => [...prev, createAgentsKilledMessage()]),
|
||||
isMessageSelectorVisible: isMessageSelectorVisible || !!showBashesDialog,
|
||||
screen,
|
||||
@@ -3018,7 +3064,7 @@ export function REPL({
|
||||
// but its controller must be reachable during preparation so Escape can
|
||||
// cancel the handoff before it dispatches a provider request.
|
||||
setAbortController(backgroundSession.abortController);
|
||||
abortController?.abort('background');
|
||||
requestBackgroundHandoffAbort(abortController);
|
||||
}, [abortController, mainLoopModel, toolPermissionContext, mainThreadAgentDefinition, getToolUseContext, customSystemPrompt, appendSystemPrompt, canUseTool, setAppState, getAutoCompactTrackingForSession, setAutoCompactTrackingForSession, fallbackModel, setAbortController, addNotification, terminalTitle]);
|
||||
const {
|
||||
handleBackgroundSession
|
||||
@@ -3266,23 +3312,11 @@ export function REPL({
|
||||
}
|
||||
}
|
||||
});
|
||||
let queryTerminal: QueryTerminal;
|
||||
let generatorDone = false;
|
||||
try {
|
||||
while (true) {
|
||||
const next = await queryGenerator.next();
|
||||
if (next.done) {
|
||||
generatorDone = true;
|
||||
queryTerminal = next.value;
|
||||
break;
|
||||
}
|
||||
const event = next.value;
|
||||
queryGuard.registerActivity(`query_event:${event.type}`, queryGeneration);
|
||||
onQueryEvent(event);
|
||||
}
|
||||
} finally {
|
||||
if (!generatorDone) await queryGenerator.return(undefined as never);
|
||||
}
|
||||
const queryTerminal = await driveQueryEvents(
|
||||
queryGenerator,
|
||||
reason => queryGuard.registerActivity(reason, queryGeneration),
|
||||
onQueryEvent,
|
||||
);
|
||||
if (isBuddyEnabled()) {
|
||||
void fireCompanionObserver(messagesRef.current, reaction => setAppState(prev => prev.companionReaction === reaction ? prev : {
|
||||
...prev,
|
||||
@@ -3359,6 +3393,20 @@ export function REPL({
|
||||
const turnBudget = turnBudgetHandoff.budget;
|
||||
foregroundTurnBudgetRef.current = turnBudgetHandoff;
|
||||
const queryContext = startResult.context;
|
||||
registerInterruptionController(abortController, {
|
||||
subsystem: 'repl',
|
||||
controllerRole: 'query-root',
|
||||
queryId: queryContext.queryId,
|
||||
queryGeneration: thisGeneration,
|
||||
querySource: queryContext.querySource,
|
||||
});
|
||||
traceInterruptionEvent('query.started', {
|
||||
subsystem: 'repl',
|
||||
phase: 'running',
|
||||
queryId: queryContext.queryId,
|
||||
queryGeneration: thisGeneration,
|
||||
querySource: queryContext.querySource,
|
||||
});
|
||||
logQueryLifecycle('start', queryContext);
|
||||
logQueryLifecycle('guard_start', queryContext);
|
||||
let didThrow = false;
|
||||
@@ -3456,8 +3504,24 @@ export function REPL({
|
||||
// then clear it only when this query reaches its terminal cleanup.
|
||||
interruptionCorrectionTracker.finishModelTurn(queryContext.queryId);
|
||||
const terminalReason = getQueryTerminalReason(abortController.signal, didThrow);
|
||||
const terminalOutcome = getQueryTerminalOutcome(abortController.signal, didThrow);
|
||||
const abortReason = getAbortReasonLabel(abortController.signal.reason);
|
||||
const activeOperations = lifecycleTracker.snapshot();
|
||||
traceInterruptionEvent('query.terminal', {
|
||||
subsystem: 'repl',
|
||||
phase: terminalReason,
|
||||
outcome: terminalOutcome,
|
||||
queryId: queryContext.queryId,
|
||||
queryGeneration: thisGeneration,
|
||||
querySource: queryContext.querySource,
|
||||
reason: abortController.signal.reason,
|
||||
causalEventId: getInterruptionSignalAbortEventId(
|
||||
abortController.signal,
|
||||
),
|
||||
activeApiCallCount: activeOperations.apiCalls.length,
|
||||
activeToolUseCount: activeOperations.toolUses.length,
|
||||
});
|
||||
if (abortController.signal.aborted) flushInterruptionTrace('query_terminal');
|
||||
const completedContext = {
|
||||
...queryContext,
|
||||
terminalReason,
|
||||
@@ -4660,7 +4724,7 @@ export function REPL({
|
||||
// (e.g. from a chat UI client via UDS).
|
||||
useEffect(() => {
|
||||
if (queuedCommands.some(cmd => cmd.priority === 'now')) {
|
||||
abortControllerRef.current?.abort('interrupt');
|
||||
requestPriorityNowAbort(abortControllerRef);
|
||||
}
|
||||
}, [queuedCommands]);
|
||||
|
||||
|
||||
@@ -3,6 +3,10 @@ import type { AppState } from '../../state/AppStateStore.js'
|
||||
import { IDLE_SPECULATION_STATE } from '../../state/AppStateStore.js'
|
||||
import type { ToolUseContext } from '../../Tool.js'
|
||||
import type { REPLHookContext } from '../../utils/hooks/postSamplingHooks.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
@@ -59,6 +63,9 @@ afterAll(() => {
|
||||
|
||||
describe('startSpeculation', () => {
|
||||
test('stops speculative writes in plan mode even when bypass is available', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
let appState = {
|
||||
speculation: IDLE_SPECULATION_STATE,
|
||||
toolPermissionContext: {
|
||||
@@ -103,8 +110,35 @@ describe('startSpeculation', () => {
|
||||
toolName: 'Write',
|
||||
},
|
||||
})
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'abort.requested',
|
||||
),
|
||||
).toMatchObject({
|
||||
source: 'speculation_edit_boundary',
|
||||
subsystem: 'prompt_suggestion',
|
||||
controllerRole: 'speculation',
|
||||
})
|
||||
abortSpeculation(setAppState)
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'abort.repeated',
|
||||
),
|
||||
).toMatchObject({
|
||||
source: 'speculation_cancelled',
|
||||
subsystem: 'prompt_suggestion',
|
||||
controllerRole: 'speculation',
|
||||
outcome: 'ignored_first_abort_wins',
|
||||
repeatedCount: 1,
|
||||
})
|
||||
} finally {
|
||||
abortSpeculation(setAppState)
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -36,6 +36,7 @@ import {
|
||||
import { formatDuration, formatNumber } from '../../utils/format.js'
|
||||
import type { REPLHookContext } from '../../utils/hooks/postSamplingHooks.js'
|
||||
import { logError } from '../../utils/log.js'
|
||||
import { requestAbort } from '../../utils/interruptionTrace.js'
|
||||
import type { SetAppState } from '../../utils/messageQueueManager.js'
|
||||
import {
|
||||
createSystemMessage,
|
||||
@@ -61,6 +62,17 @@ import {
|
||||
const MAX_SPECULATION_TURNS = 20
|
||||
const MAX_SPECULATION_MESSAGES = 100
|
||||
|
||||
function requestSpeculationAbort(
|
||||
controller: AbortController,
|
||||
source: string,
|
||||
): void {
|
||||
requestAbort(controller, undefined, {
|
||||
source,
|
||||
subsystem: 'prompt_suggestion',
|
||||
controllerRole: 'speculation',
|
||||
})
|
||||
}
|
||||
|
||||
const WRITE_TOOLS = new Set(['Edit', 'Write', 'NotebookEdit'])
|
||||
const SAFE_READ_ONLY_TOOLS = new Set([
|
||||
'Read',
|
||||
@@ -371,6 +383,11 @@ async function generatePipelinedSuggestion(
|
||||
|
||||
const pipelineAbortController = createChildAbortController(
|
||||
parentAbortController,
|
||||
undefined,
|
||||
{
|
||||
subsystem: 'prompt_suggestion',
|
||||
controllerRole: 'suggestion-pipeline',
|
||||
},
|
||||
)
|
||||
if (pipelineAbortController.signal.aborted) return
|
||||
|
||||
@@ -418,6 +435,11 @@ export async function startSpeculation(
|
||||
|
||||
const abortController = createChildAbortController(
|
||||
context.toolUseContext.abortController,
|
||||
undefined,
|
||||
{
|
||||
subsystem: 'prompt_suggestion',
|
||||
controllerRole: 'speculation',
|
||||
},
|
||||
)
|
||||
|
||||
if (abortController.signal.aborted) return
|
||||
@@ -442,7 +464,7 @@ export async function startSpeculation(
|
||||
speculation: {
|
||||
status: 'active',
|
||||
id,
|
||||
abort: () => abortController.abort(),
|
||||
abort: () => requestSpeculationAbort(abortController, 'speculation_cancelled'),
|
||||
startTime,
|
||||
messagesRef,
|
||||
writtenPathsRef,
|
||||
@@ -495,7 +517,10 @@ export async function startSpeculation(
|
||||
completedAt: Date.now(),
|
||||
},
|
||||
}))
|
||||
abortController.abort()
|
||||
requestSpeculationAbort(
|
||||
abortController,
|
||||
'speculation_edit_boundary',
|
||||
)
|
||||
return denySpeculation(
|
||||
'Speculation paused: file edit requires permission',
|
||||
'speculation_edit_boundary',
|
||||
@@ -599,7 +624,10 @@ export async function startSpeculation(
|
||||
updateActiveSpeculationState(setAppState, () => ({
|
||||
boundary: { type: 'bash', command, completedAt: Date.now() },
|
||||
}))
|
||||
abortController.abort()
|
||||
requestSpeculationAbort(
|
||||
abortController,
|
||||
'speculation_bash_boundary',
|
||||
)
|
||||
return denySpeculation(
|
||||
'Speculation paused: bash boundary',
|
||||
'speculation_bash_boundary',
|
||||
@@ -633,7 +661,7 @@ export async function startSpeculation(
|
||||
completedAt: Date.now(),
|
||||
},
|
||||
}))
|
||||
abortController.abort()
|
||||
requestSpeculationAbort(abortController, 'speculation_unknown_tool')
|
||||
return denySpeculation(
|
||||
`Tool ${tool.name} not allowed during speculation`,
|
||||
'speculation_unknown_tool',
|
||||
@@ -647,7 +675,10 @@ export async function startSpeculation(
|
||||
if (msg.type === 'assistant' || msg.type === 'user') {
|
||||
messagesRef.current.push(msg)
|
||||
if (messagesRef.current.length >= MAX_SPECULATION_MESSAGES) {
|
||||
abortController.abort()
|
||||
requestSpeculationAbort(
|
||||
abortController,
|
||||
'speculation_message_limit',
|
||||
)
|
||||
}
|
||||
if (isUserMessageWithArrayContent(msg)) {
|
||||
const newTools = count(
|
||||
@@ -687,7 +718,7 @@ export async function startSpeculation(
|
||||
abortController,
|
||||
)
|
||||
} catch (error) {
|
||||
abortController.abort()
|
||||
requestSpeculationAbort(abortController, 'speculation_failure')
|
||||
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
safeRemoveOverlay(overlayPath)
|
||||
|
||||
@@ -83,6 +83,19 @@ describe('Claude stream abort classification wiring', () => {
|
||||
)
|
||||
})
|
||||
|
||||
test('does not infer user intent from an external AbortError while root is active', () => {
|
||||
const activeRoot = new AbortController()
|
||||
const externalAbort = new APIUserAbortError()
|
||||
|
||||
expect(activeRoot.signal.aborted).toBe(false)
|
||||
expect(shouldCreateUserInterruptionMessage(activeRoot.signal.reason)).toBe(
|
||||
false,
|
||||
)
|
||||
expect(
|
||||
getClaudeStreamingAbortLogMessage(activeRoot.signal, externalAbort),
|
||||
).toBe('Streaming aborted by parent signal: Request was aborted.')
|
||||
})
|
||||
|
||||
test('labels expected side-task retry aborts without reclassifying user aborts', () => {
|
||||
const expectedAbort = new AbortController()
|
||||
expectedAbort.abort('agent-summary-superseded')
|
||||
|
||||
@@ -14,6 +14,10 @@ import { resetGrowthBook } from '../analytics/growthbook.js'
|
||||
import { getEmptyToolPermissionContext } from '../../Tool.js'
|
||||
import type { Message } from '../../types/message.js'
|
||||
import { QueryLifecycleOperationTracker } from '../../utils/queryLifecycle.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
import { asSystemPrompt } from '../../utils/systemPromptType.js'
|
||||
import {
|
||||
executeNonStreamingRequest,
|
||||
@@ -39,6 +43,7 @@ const envKeys = [
|
||||
'CLAUDE_CODE_USE_OPENAI',
|
||||
'CLAUDE_CODE_USE_VERTEX',
|
||||
'CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK',
|
||||
'CLAUDE_DISABLE_STREAM_WATCHDOG',
|
||||
'CLAUDE_FEATURE_FLAGS_FILE',
|
||||
'CLAUDE_STREAM_IDLE_TIMEOUT_MS',
|
||||
'GEMINI_API_KEY',
|
||||
@@ -46,6 +51,7 @@ const envKeys = [
|
||||
'OPENAI_API_KEY',
|
||||
'OPENAI_BASE_URL',
|
||||
'OPENAI_MODEL',
|
||||
'OPENCLAUDE_INTERRUPT_TRACE',
|
||||
'OPENCLAUDE_MAX_RETRIES',
|
||||
'VCR_RECORD',
|
||||
] as const
|
||||
@@ -490,6 +496,8 @@ describe('Claude API lifecycle tracking', () => {
|
||||
test('preserves provider override and query source during 404 non-streaming fallback', async () => {
|
||||
setClientTestEnv()
|
||||
process.env.OPENCLAUDE_MAX_RETRIES = '0'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const queryLifecycle = new QueryLifecycleOperationTracker()
|
||||
const providerBaseURL = 'https://provider.example/v1'
|
||||
const requests: {
|
||||
@@ -541,6 +549,13 @@ describe('Claude API lifecycle tracking', () => {
|
||||
|
||||
for await (const message of generator) {
|
||||
messages.push(message)
|
||||
if (
|
||||
typeof message === 'object' &&
|
||||
message !== null &&
|
||||
(message as { type?: unknown }).type === 'assistant'
|
||||
) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
const streamingRequest = requests.find(request => request.stream === true)
|
||||
@@ -562,6 +577,27 @@ describe('Claude API lifecycle tracking', () => {
|
||||
querySource: 'sdk',
|
||||
})
|
||||
expect(queryLifecycle.snapshot().apiCalls).toEqual([])
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const creationError = trace.find(
|
||||
entry =>
|
||||
entry.event === 'claude_stream.error' &&
|
||||
entry.phase === 'stream_creation',
|
||||
)
|
||||
const fallbackStarted = trace.find(
|
||||
entry => entry.event === 'claude_stream.fallback_started',
|
||||
)
|
||||
const fallbackSettled = trace.find(
|
||||
entry => entry.event === 'claude_stream.fallback_settled',
|
||||
)
|
||||
expect(creationError).toBeDefined()
|
||||
expect(fallbackStarted).toMatchObject({
|
||||
trigger: '404_stream_creation',
|
||||
causalEventId: creationError?.eventId,
|
||||
})
|
||||
expect(fallbackSettled).toMatchObject({
|
||||
outcome: 'completed',
|
||||
causalEventId: fallbackStarted?.eventId,
|
||||
})
|
||||
})
|
||||
|
||||
test('parent abort during OpenAI-compatible stream does not start non-streaming fallback', async () => {
|
||||
@@ -658,9 +694,12 @@ describe('Claude API lifecycle tracking', () => {
|
||||
|
||||
test('stream idle timeout respects disabled non-streaming fallback guard', async () => {
|
||||
setClientTestEnv()
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
process.env.OPENCLAUDE_MAX_RETRIES = '0'
|
||||
process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = String(TEST_STREAM_IDLE_TIMEOUT_MS)
|
||||
process.env.CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK = '1'
|
||||
process.env.CLAUDE_DISABLE_STREAM_WATCHDOG = '1'
|
||||
const queryLifecycle = new QueryLifecycleOperationTracker()
|
||||
const parent = new AbortController()
|
||||
let fallbackRequests = 0
|
||||
@@ -726,6 +765,17 @@ describe('Claude API lifecycle tracking', () => {
|
||||
JSON.stringify((message as { message?: { content?: unknown } }).message?.content).includes('Stream idle timeout'),
|
||||
),
|
||||
).toBe(true)
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const providerIdle = trace.find(
|
||||
entry =>
|
||||
entry.event === 'provider_stream.idle_timeout' &&
|
||||
entry.transport === 'openai_chat_completions',
|
||||
)
|
||||
const claudeError = trace.find(
|
||||
entry => entry.event === 'claude_stream.error',
|
||||
)
|
||||
expect(providerIdle).toBeDefined()
|
||||
expect(claudeError?.causalEventId).toBe(providerIdle?.eventId)
|
||||
})
|
||||
|
||||
test('tracks each non-streaming fallback request and clears it on success', async () => {
|
||||
|
||||
@@ -11,7 +11,7 @@ import type {
|
||||
BetaRawMessageStreamEvent,
|
||||
} from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs'
|
||||
import type { Stream } from '@anthropic-ai/sdk/streaming.mjs'
|
||||
import { mkdtempSync, rmSync } from 'fs'
|
||||
import { mkdtempSync, readFileSync, rmSync } from 'fs'
|
||||
import { tmpdir } from 'os'
|
||||
import { join } from 'path'
|
||||
import {
|
||||
@@ -22,6 +22,13 @@ import { getEmptyToolPermissionContext } from '../../Tool.js'
|
||||
import type { Message } from '../../types/message.js'
|
||||
import { asSystemPrompt } from '../../utils/systemPromptType.js'
|
||||
import { QueryLifecycleOperationTracker } from '../../utils/queryLifecycle.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
registerInterruptionController,
|
||||
requestAbort,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
import { EMPTY_USAGE } from './emptyUsage.js'
|
||||
import type { Options } from './claude.js'
|
||||
|
||||
@@ -39,6 +46,8 @@ const envKeys = [
|
||||
'CLAUDE_ENABLE_STREAM_WATCHDOG',
|
||||
'CLAUDE_STREAM_IDLE_TIMEOUT_MS',
|
||||
'OPENCLAUDE_MAX_RETRIES',
|
||||
'OPENCLAUDE_INTERRUPT_TRACE',
|
||||
'OPENCLAUDE_INTERRUPT_TRACE_FILE',
|
||||
'VCR_RECORD',
|
||||
] as const
|
||||
|
||||
@@ -295,6 +304,8 @@ beforeEach(async () => {
|
||||
await acquireSharedMutationLock('claude.streamWatchdog.test.ts')
|
||||
installClientSpy()
|
||||
setTestMacro()
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
for (const key of envKeys) {
|
||||
delete process.env[key]
|
||||
}
|
||||
@@ -306,11 +317,13 @@ beforeEach(async () => {
|
||||
process.env.VCR_RECORD = '1'
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
afterEach(async () => {
|
||||
try {
|
||||
restoreClientSpy?.()
|
||||
restoreClientSpy = undefined
|
||||
createHandler = undefined
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
for (const key of envKeys) {
|
||||
const envKey: string = key
|
||||
if (
|
||||
@@ -343,6 +356,9 @@ afterEach(() => {
|
||||
|
||||
describe('Claude stream watchdog', () => {
|
||||
test('falls back when the top-level stream iterator never settles', async () => {
|
||||
const traceFile = join(fixturesRoot!, 'interruption-trace.jsonl')
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = traceFile
|
||||
const wedged = makeWedgedStream()
|
||||
const streamModes: unknown[] = []
|
||||
createHandler = params => {
|
||||
@@ -369,6 +385,64 @@ describe('Claude stream watchdog', () => {
|
||||
expect(streamModes).toEqual([true, undefined])
|
||||
expect(wedged.abortSignal.aborted).toBe(true)
|
||||
expect(wedged.returnCalled()).toBe(true)
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
const trace = readFileSync(traceFile, 'utf8')
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(line => JSON.parse(line) as {
|
||||
event: string
|
||||
eventId?: string
|
||||
causalEventId?: string
|
||||
source?: string
|
||||
outcome?: string
|
||||
})
|
||||
const events = trace.map(entry => entry.event)
|
||||
expect(events.indexOf('claude_stream.idle_timeout')).toBeGreaterThanOrEqual(0)
|
||||
expect(events.indexOf('claude_stream.loop_settled')).toBeGreaterThan(
|
||||
events.indexOf('claude_stream.idle_timeout'),
|
||||
)
|
||||
expect(events.indexOf('claude_stream.fallback_started')).toBeGreaterThan(
|
||||
events.indexOf('claude_stream.loop_settled'),
|
||||
)
|
||||
expect(trace).toContainEqual(
|
||||
expect.objectContaining({
|
||||
event: 'abort.requested',
|
||||
source: 'claude_stream_watchdog',
|
||||
}),
|
||||
)
|
||||
const idleTimeout = trace.find(
|
||||
entry => entry.event === 'claude_stream.idle_timeout',
|
||||
)
|
||||
const providerAbort = trace.find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.source === 'claude_stream_watchdog',
|
||||
)
|
||||
const loopSettled = trace.find(
|
||||
entry => entry.event === 'claude_stream.loop_settled',
|
||||
)
|
||||
const fallbackStarted = trace.find(
|
||||
entry => entry.event === 'claude_stream.fallback_started',
|
||||
)
|
||||
const fallbackSettled = trace.find(
|
||||
entry => entry.event === 'claude_stream.fallback_settled',
|
||||
)
|
||||
expect(idleTimeout).toBeDefined()
|
||||
expect(providerAbort).toBeDefined()
|
||||
expect(loopSettled).toBeDefined()
|
||||
expect(fallbackStarted).toBeDefined()
|
||||
expect(fallbackSettled).toBeDefined()
|
||||
expect(typeof idleTimeout!.eventId).toBe('string')
|
||||
expect(typeof providerAbort!.causalEventId).toBe('string')
|
||||
expect(typeof loopSettled!.causalEventId).toBe('string')
|
||||
expect(typeof fallbackStarted!.causalEventId).toBe('string')
|
||||
expect(providerAbort!.causalEventId).toBe(idleTimeout!.eventId)
|
||||
expect(loopSettled!.causalEventId).toBe(idleTimeout!.eventId)
|
||||
expect(fallbackStarted!.causalEventId).toBe(idleTimeout!.eventId)
|
||||
expect(fallbackSettled).toMatchObject({
|
||||
causalEventId: fallbackStarted!.eventId,
|
||||
outcome: 'completed',
|
||||
})
|
||||
expect(
|
||||
(result as unknown[]).some(
|
||||
message =>
|
||||
@@ -385,8 +459,10 @@ describe('Claude stream watchdog', () => {
|
||||
|
||||
test('does not attempt fallback when the parent signal aborts first', async () => {
|
||||
process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = '250'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const wedged = makeWedgedStream()
|
||||
const controller = new AbortController()
|
||||
registerInterruptionController(controller, { controllerRole: 'query-root' })
|
||||
const streamModes: unknown[] = []
|
||||
createHandler = params => {
|
||||
streamModes.push(params.stream)
|
||||
@@ -406,7 +482,10 @@ describe('Claude stream watchdog', () => {
|
||||
error instanceof Error ? error.name : String(error),
|
||||
)
|
||||
await wedged.nextStarted
|
||||
controller.abort()
|
||||
requestAbort(controller, undefined, {
|
||||
source: 'cancel_keybinding',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
|
||||
try {
|
||||
const result = await Promise.race([request, delay(150)])
|
||||
@@ -414,6 +493,97 @@ describe('Claude stream watchdog', () => {
|
||||
expect(streamModes).toEqual([true])
|
||||
expect(wedged.abortSignal.aborted).toBe(true)
|
||||
expect(wedged.returnCalled()).toBe(true)
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const rootAbort = trace.find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.controllerRole === 'query-root',
|
||||
)
|
||||
const parentAbort = trace.find(
|
||||
entry => entry.event === 'claude_stream.parent_abort',
|
||||
)
|
||||
const providerAbort = trace.find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.source === 'claude_stream_parent',
|
||||
)
|
||||
expect(rootAbort).toBeDefined()
|
||||
expect(parentAbort).toBeDefined()
|
||||
expect(providerAbort).toBeDefined()
|
||||
expect(typeof rootAbort!.eventId).toBe('string')
|
||||
expect(typeof parentAbort!.eventId).toBe('string')
|
||||
expect(typeof parentAbort!.causalEventId).toBe('string')
|
||||
expect(typeof providerAbort!.causalEventId).toBe('string')
|
||||
expect(parentAbort!.causalEventId).toBe(rootAbort!.eventId)
|
||||
expect(providerAbort!.causalEventId).toBe(parentAbort!.eventId)
|
||||
} finally {
|
||||
wedged.rejectPendingNext(new Error('test cleanup'))
|
||||
await settleForCleanup(request)
|
||||
}
|
||||
})
|
||||
|
||||
test('records a terminal failed outcome when non-streaming fallback rejects', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const wedged = makeWedgedStream()
|
||||
createHandler = params => {
|
||||
if (params.stream === true) return makeWithResponse(wedged.stream)
|
||||
return Promise.reject(new Error('synthetic fallback failure'))
|
||||
}
|
||||
|
||||
const request = collectStreamingMessages(
|
||||
new AbortController().signal,
|
||||
makeOptions(),
|
||||
)
|
||||
await wedged.nextStarted
|
||||
try {
|
||||
await Promise.race([request, delay(250)])
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const started = trace.find(
|
||||
entry => entry.event === 'claude_stream.fallback_started',
|
||||
)
|
||||
const settled = trace.find(
|
||||
entry => entry.event === 'claude_stream.fallback_settled',
|
||||
)
|
||||
expect(started).toBeDefined()
|
||||
expect(settled).toMatchObject({
|
||||
outcome: 'failed',
|
||||
causalEventId: started!.eventId,
|
||||
})
|
||||
} finally {
|
||||
wedged.rejectPendingNext(new Error('test cleanup'))
|
||||
await settleForCleanup(request)
|
||||
}
|
||||
})
|
||||
|
||||
test('records fallback failure when the returned message cannot be normalized', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const wedged = makeWedgedStream()
|
||||
createHandler = params => {
|
||||
if (params.stream === true) return makeWithResponse(wedged.stream)
|
||||
return Promise.resolve(
|
||||
makeBetaMessage('msg-invalid-fallback', [null] as never),
|
||||
)
|
||||
}
|
||||
|
||||
const request = collectStreamingMessages(
|
||||
new AbortController().signal,
|
||||
makeOptions(),
|
||||
)
|
||||
await wedged.nextStarted
|
||||
try {
|
||||
const result = await Promise.race([
|
||||
request.then(
|
||||
() => 'resolved',
|
||||
error => error,
|
||||
),
|
||||
delay(250),
|
||||
])
|
||||
expect(result).not.toBe('timeout')
|
||||
const settlements = __getInterruptionTraceSnapshotForTests().filter(
|
||||
entry => entry.event === 'claude_stream.fallback_settled',
|
||||
)
|
||||
expect(settlements).toHaveLength(1)
|
||||
expect(settlements[0]?.outcome).toBe('failed')
|
||||
} finally {
|
||||
wedged.rejectPendingNext(new Error('test cleanup'))
|
||||
await settleForCleanup(request)
|
||||
|
||||
+179
-13
@@ -175,6 +175,13 @@ import { CHROME_TOOL_SEARCH_INSTRUCTIONS } from 'src/utils/claudeInChrome/prompt
|
||||
import { COMPACT_MAX_OUTPUT_TOKENS, getContextWindowForModel, getMaxThinkingTokensForModel } from 'src/utils/context.js'
|
||||
import { logForDebugging } from 'src/utils/debug.js'
|
||||
import { logForDiagnosticsNoPII } from 'src/utils/diagLogs.js'
|
||||
import {
|
||||
flushInterruptionTrace,
|
||||
getInterruptionErrorCausalEventId,
|
||||
getInterruptionSignalAbortEventId,
|
||||
requestAbort,
|
||||
traceInterruptionEvent,
|
||||
} from 'src/utils/interruptionTrace.js'
|
||||
import { type EffortValue, modelSupportsEffort } from 'src/utils/effort.js'
|
||||
import type { QueryLifecycleOperationTracker } from 'src/utils/queryLifecycle.js'
|
||||
import {
|
||||
@@ -1157,6 +1164,21 @@ async function* queryModel(
|
||||
void
|
||||
> {
|
||||
const providerRequestModel = options.requestModel ?? options.model
|
||||
function traceFallbackSettlement(
|
||||
outcome: 'superseded' | 'aborted' | 'failed' | 'completed',
|
||||
causalEventId: string | undefined,
|
||||
error?: unknown,
|
||||
): void {
|
||||
traceInterruptionEvent('claude_stream.fallback_settled', {
|
||||
subsystem: 'claude_stream',
|
||||
transport: 'anthropic_messages',
|
||||
model: options.model,
|
||||
outcome,
|
||||
causalEventId,
|
||||
...(error === undefined ? {} : { error }),
|
||||
})
|
||||
flushInterruptionTrace('claude_stream_fallback_settled')
|
||||
}
|
||||
// Check cheap conditions first — the off-switch await blocks on GrowthBook
|
||||
// init (~10ms). For non-Opus models (haiku, sonnet) this skips the await
|
||||
// entirely. Subscribers don't hit this path at all.
|
||||
@@ -2101,6 +2123,7 @@ async function* queryModel(
|
||||
const STREAM_IDLE_TIMEOUT_MS = getStreamIdleTimeoutMs()
|
||||
const STREAM_IDLE_WARNING_MS = STREAM_IDLE_TIMEOUT_MS / 2
|
||||
let streamIdleAborted = false
|
||||
let streamSettlementCausalEventId: string | undefined
|
||||
// performance.now() snapshot when watchdog fires, for measuring abort propagation delay
|
||||
let streamWatchdogFiredAt: number | null = null
|
||||
let streamIdleWarningTimer: ReturnType<typeof setTimeout> | null = null
|
||||
@@ -2123,19 +2146,36 @@ async function* queryModel(
|
||||
{ level: 'warn' },
|
||||
)
|
||||
logForDiagnosticsNoPII('warn', 'cli_streaming_idle_warning')
|
||||
traceInterruptionEvent('claude_stream.idle_warning', {
|
||||
subsystem: 'claude_stream',
|
||||
transport: 'anthropic_messages',
|
||||
model: options.model,
|
||||
sinceLastYieldMs: warnMs,
|
||||
})
|
||||
}
|
||||
|
||||
function closeStreamIterator(
|
||||
iterator: AsyncIterator<BetaRawMessageStreamEvent>,
|
||||
reason: Error,
|
||||
source: 'claude_stream_watchdog' | 'claude_stream_parent',
|
||||
causalEventId?: string,
|
||||
): void {
|
||||
const activeStream = stream
|
||||
releaseStreamResources()
|
||||
try {
|
||||
activeStream?.controller?.abort(reason)
|
||||
if (activeStream?.controller) {
|
||||
requestAbort(activeStream.controller, reason, {
|
||||
source,
|
||||
causalEventId,
|
||||
subsystem: 'claude_stream',
|
||||
transport: 'anthropic_messages',
|
||||
model: options.model,
|
||||
controllerRole: 'provider-stream',
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
// Ignore - the stream may already be closed by the SDK.
|
||||
}
|
||||
releaseStreamResources()
|
||||
|
||||
try {
|
||||
const returned = iterator.return?.()
|
||||
@@ -2158,6 +2198,14 @@ async function* queryModel(
|
||||
{ level: 'error' },
|
||||
)
|
||||
logForDiagnosticsNoPII('error', 'cli_streaming_idle_timeout')
|
||||
const causalEventId = traceInterruptionEvent('claude_stream.idle_timeout', {
|
||||
subsystem: 'claude_stream',
|
||||
transport: 'anthropic_messages',
|
||||
model: options.model,
|
||||
sinceLastYieldMs: STREAM_IDLE_TIMEOUT_MS,
|
||||
})
|
||||
streamSettlementCausalEventId = causalEventId
|
||||
flushInterruptionTrace('claude_stream_idle_timeout')
|
||||
logEvent('tengu_streaming_idle_timeout', {
|
||||
model:
|
||||
options.model as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
@@ -2166,7 +2214,12 @@ async function* queryModel(
|
||||
timeout_ms: STREAM_IDLE_TIMEOUT_MS,
|
||||
})
|
||||
|
||||
closeStreamIterator(iterator, timeoutError)
|
||||
closeStreamIterator(
|
||||
iterator,
|
||||
timeoutError,
|
||||
'claude_stream_watchdog',
|
||||
causalEventId,
|
||||
)
|
||||
}
|
||||
|
||||
function readNextStreamPart(
|
||||
@@ -2174,7 +2227,14 @@ async function* queryModel(
|
||||
): Promise<IteratorResult<BetaRawMessageStreamEvent>> {
|
||||
if (signal.aborted) {
|
||||
const abortError = new APIUserAbortError()
|
||||
closeStreamIterator(iterator, abortError)
|
||||
streamSettlementCausalEventId =
|
||||
getInterruptionSignalAbortEventId(signal)
|
||||
closeStreamIterator(
|
||||
iterator,
|
||||
abortError,
|
||||
'claude_stream_parent',
|
||||
getInterruptionSignalAbortEventId(signal),
|
||||
)
|
||||
return Promise.reject(abortError)
|
||||
}
|
||||
|
||||
@@ -2200,7 +2260,22 @@ async function* queryModel(
|
||||
}
|
||||
const onAbort = () => {
|
||||
const abortError = new APIUserAbortError()
|
||||
closeStreamIterator(iterator, abortError)
|
||||
const parentCausalEventId = getInterruptionSignalAbortEventId(signal)
|
||||
const causalEventId = traceInterruptionEvent('claude_stream.parent_abort', {
|
||||
subsystem: 'claude_stream',
|
||||
transport: 'anthropic_messages',
|
||||
model: options.model,
|
||||
reason: signal.reason,
|
||||
causalEventId: parentCausalEventId,
|
||||
})
|
||||
streamSettlementCausalEventId =
|
||||
causalEventId ?? parentCausalEventId
|
||||
closeStreamIterator(
|
||||
iterator,
|
||||
abortError,
|
||||
'claude_stream_parent',
|
||||
causalEventId ?? parentCausalEventId,
|
||||
)
|
||||
settleReject(abortError)
|
||||
}
|
||||
|
||||
@@ -2630,6 +2705,14 @@ async function* queryModel(
|
||||
streamWatchdogFiredAt !== null
|
||||
? Math.round(performance.now() - streamWatchdogFiredAt)
|
||||
: -1
|
||||
traceInterruptionEvent('claude_stream.loop_settled', {
|
||||
subsystem: 'claude_stream',
|
||||
transport: 'anthropic_messages',
|
||||
model: options.model,
|
||||
outcome: 'clean',
|
||||
causalEventId: streamSettlementCausalEventId,
|
||||
sinceLastYieldMs: exitDelayMs,
|
||||
})
|
||||
logForDiagnosticsNoPII(
|
||||
'info',
|
||||
'cli_stream_loop_exited_after_watchdog_clean',
|
||||
@@ -2719,6 +2802,20 @@ async function* queryModel(
|
||||
} catch (streamingError) {
|
||||
// Clear the idle timeout watchdog on error path too
|
||||
clearStreamIdleTimers()
|
||||
streamSettlementCausalEventId =
|
||||
getInterruptionErrorCausalEventId(streamingError) ??
|
||||
streamSettlementCausalEventId
|
||||
traceInterruptionEvent('claude_stream.error', {
|
||||
subsystem: 'claude_stream',
|
||||
transport: 'anthropic_messages',
|
||||
model: options.model,
|
||||
outcome: signal.aborted ? 'root_aborted' : 'external_error',
|
||||
reason: signal.reason,
|
||||
causalEventId:
|
||||
getInterruptionSignalAbortEventId(signal) ??
|
||||
streamSettlementCausalEventId,
|
||||
error: streamingError,
|
||||
})
|
||||
|
||||
// Instrumentation: if the watchdog had already fired and the for-await
|
||||
// threw (rather than exiting cleanly), record that the loop DID exit and
|
||||
@@ -2727,6 +2824,15 @@ async function* queryModel(
|
||||
const exitDelayMs = Math.round(
|
||||
performance.now() - streamWatchdogFiredAt,
|
||||
)
|
||||
traceInterruptionEvent('claude_stream.loop_settled', {
|
||||
subsystem: 'claude_stream',
|
||||
transport: 'anthropic_messages',
|
||||
model: options.model,
|
||||
outcome: 'error',
|
||||
causalEventId: streamSettlementCausalEventId,
|
||||
error: streamingError,
|
||||
sinceLastYieldMs: exitDelayMs,
|
||||
})
|
||||
logForDiagnosticsNoPII(
|
||||
'info',
|
||||
'cli_stream_loop_exited_after_watchdog_error',
|
||||
@@ -2862,6 +2968,16 @@ async function* queryModel(
|
||||
// Instrumentation: proves executeNonStreamingRequest was entered (vs. the
|
||||
// fallback event firing but the call itself hanging at dispatch).
|
||||
logForDiagnosticsNoPII('info', 'cli_nonstreaming_fallback_started')
|
||||
const fallbackStartedEventId = traceInterruptionEvent(
|
||||
'claude_stream.fallback_started', {
|
||||
subsystem: 'claude_stream',
|
||||
transport: 'anthropic_messages',
|
||||
model: options.model,
|
||||
trigger: streamIdleAborted ? 'watchdog' : 'other',
|
||||
causalEventId: streamSettlementCausalEventId,
|
||||
},
|
||||
)
|
||||
flushInterruptionTrace('claude_stream_fallback_started')
|
||||
logEvent('tengu_nonstreaming_fallback_started', {
|
||||
request_id: (streamRequestId ??
|
||||
'unknown') as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
@@ -2872,7 +2988,10 @@ async function* queryModel(
|
||||
: 'other') as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
})
|
||||
endActiveApiCall()
|
||||
const result = yield* executeNonStreamingRequest(
|
||||
let result: BetaMessage | null
|
||||
let fallbackResultMessage: AssistantMessage | undefined
|
||||
try {
|
||||
result = yield* executeNonStreamingRequest(
|
||||
{ model: providerRequestModel, source: options.querySource, providerOverride: options.providerOverride, effortValue: effort },
|
||||
{
|
||||
model: providerRequestModel,
|
||||
@@ -2894,9 +3013,12 @@ async function* queryModel(
|
||||
options.onProviderRequestStart,
|
||||
)
|
||||
|
||||
if (result === null) return
|
||||
if (result === null) {
|
||||
traceFallbackSettlement('superseded', fallbackStartedEventId)
|
||||
return
|
||||
}
|
||||
|
||||
const m: AssistantMessage = {
|
||||
fallbackResultMessage = {
|
||||
message: {
|
||||
...result,
|
||||
content: normalizeContentFromAPI(
|
||||
@@ -2917,9 +3039,18 @@ async function* queryModel(
|
||||
advisorModel,
|
||||
}),
|
||||
}
|
||||
newMessages.push(m)
|
||||
fallbackMessage = m
|
||||
yield m
|
||||
newMessages.push(fallbackResultMessage)
|
||||
fallbackMessage = fallbackResultMessage
|
||||
} catch (error) {
|
||||
traceFallbackSettlement(
|
||||
signal.aborted ? 'aborted' : 'failed',
|
||||
fallbackStartedEventId,
|
||||
error,
|
||||
)
|
||||
throw error
|
||||
}
|
||||
traceFallbackSettlement('completed', fallbackStartedEventId)
|
||||
yield fallbackResultMessage
|
||||
} finally {
|
||||
clearStreamIdleTimers()
|
||||
}
|
||||
@@ -2959,6 +3090,16 @@ async function* queryModel(
|
||||
errorFromRetry.originalError.status === 404
|
||||
|
||||
if (is404StreamCreationError) {
|
||||
const streamCreationErrorEventId = traceInterruptionEvent(
|
||||
'claude_stream.error',
|
||||
{
|
||||
subsystem: 'claude_stream',
|
||||
phase: 'stream_creation',
|
||||
transport: 'anthropic_messages',
|
||||
model: options.model,
|
||||
error: errorFromRetry,
|
||||
},
|
||||
)
|
||||
// 404 is thrown at .withResponse() before streamRequestId is assigned,
|
||||
// and CannotRetryError means every retry failed — so grab the failed
|
||||
// request's ID from the error header instead.
|
||||
@@ -2988,6 +3129,18 @@ async function* queryModel(
|
||||
'404_stream_creation' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
})
|
||||
|
||||
const fallbackStartedEventId = traceInterruptionEvent(
|
||||
'claude_stream.fallback_started',
|
||||
{
|
||||
subsystem: 'claude_stream',
|
||||
transport: 'anthropic_messages',
|
||||
model: options.model,
|
||||
trigger: '404_stream_creation',
|
||||
causalEventId: streamCreationErrorEventId,
|
||||
},
|
||||
)
|
||||
flushInterruptionTrace('claude_stream_fallback_started')
|
||||
|
||||
try {
|
||||
// Fall back to non-streaming mode
|
||||
endActiveApiCall()
|
||||
@@ -3017,7 +3170,10 @@ async function* queryModel(
|
||||
options.onProviderRequestStart,
|
||||
)
|
||||
|
||||
if (result === null) return
|
||||
if (result === null) {
|
||||
traceFallbackSettlement('superseded', fallbackStartedEventId)
|
||||
return
|
||||
}
|
||||
|
||||
const m: AssistantMessage = {
|
||||
message: {
|
||||
@@ -3038,10 +3194,16 @@ async function* queryModel(
|
||||
}
|
||||
newMessages.push(m)
|
||||
fallbackMessage = m
|
||||
traceFallbackSettlement('completed', fallbackStartedEventId)
|
||||
yield m
|
||||
|
||||
// Continue to success logging below
|
||||
} catch (fallbackError) {
|
||||
traceFallbackSettlement(
|
||||
signal.aborted ? 'aborted' : 'failed',
|
||||
fallbackStartedEventId,
|
||||
fallbackError,
|
||||
)
|
||||
// Propagate model-fallback signal to query.ts (see comment above).
|
||||
if (fallbackError instanceof FallbackTriggeredError) {
|
||||
throw fallbackError
|
||||
@@ -3265,7 +3427,11 @@ export function cleanupStream(
|
||||
try {
|
||||
// Abort the stream via its controller if not already aborted
|
||||
if (!stream.controller.signal.aborted) {
|
||||
stream.controller.abort()
|
||||
requestAbort(stream.controller, undefined, {
|
||||
source: 'claude_stream_cleanup',
|
||||
subsystem: 'claude_stream',
|
||||
controllerRole: 'provider-stream',
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
// Ignore - stream may already be closed
|
||||
|
||||
@@ -0,0 +1,841 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../../test/sharedMutationLock.js'
|
||||
import type { AnthropicStreamEvent } from './codexShim.js'
|
||||
import {
|
||||
__codexStreamToAnthropicForTests,
|
||||
codexStreamToAnthropic,
|
||||
} from './codexShim.js'
|
||||
import { QueryGuard, type QueryGuardTimeoutReason } from '../../utils/QueryGuard.js'
|
||||
import type { QueryGuardTimeoutInfo } from '../../utils/queryLifecycle.js'
|
||||
import { driveQueryEvents } from '../../utils/queryEventDriver.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
requestAbort,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock('codexShim.interruption.test.ts')
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
} finally {
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
async function bounded<T>(promise: Promise<T>, timeoutMs = 2_000): Promise<T> {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
try {
|
||||
return await Promise.race([
|
||||
promise,
|
||||
new Promise<T>((_, reject) => {
|
||||
timer = setTimeout(
|
||||
() => reject(new Error('issue-1830 test did not settle')),
|
||||
timeoutMs,
|
||||
)
|
||||
}),
|
||||
])
|
||||
} finally {
|
||||
if (timer !== undefined) clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
|
||||
function responseFromText(text: string): Response {
|
||||
const encoder = new TextEncoder()
|
||||
return new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(text))
|
||||
controller.close()
|
||||
},
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
async function collectCodex(response: Response): Promise<AnthropicStreamEvent[]> {
|
||||
const events: AnthropicStreamEvent[] = []
|
||||
for await (const event of codexStreamToAnthropic(response, 'gpt-test')) {
|
||||
events.push(event)
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
function makeTimedStream(
|
||||
makeFrame: (index: number) => string,
|
||||
intervalMs: number,
|
||||
): {
|
||||
response: Response
|
||||
cancelReasons: unknown[]
|
||||
getEmissionCount: () => number
|
||||
stop: () => void
|
||||
} {
|
||||
const cancelReasons: unknown[] = []
|
||||
const encoder = new TextEncoder()
|
||||
let index = 0
|
||||
let timer: ReturnType<typeof setInterval> | undefined
|
||||
let stopped = false
|
||||
let streamController: ReadableStreamDefaultController<Uint8Array> | undefined
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
streamController = controller
|
||||
timer = setInterval(() => {
|
||||
if (stopped) return
|
||||
try {
|
||||
controller.enqueue(encoder.encode(makeFrame(index++)))
|
||||
} catch {
|
||||
stopped = true
|
||||
}
|
||||
}, intervalMs)
|
||||
},
|
||||
cancel(reason) {
|
||||
stopped = true
|
||||
if (timer !== undefined) clearInterval(timer)
|
||||
cancelReasons.push(reason)
|
||||
},
|
||||
})
|
||||
return {
|
||||
response: new Response(stream),
|
||||
cancelReasons,
|
||||
getEmissionCount: () => index,
|
||||
stop: () => {
|
||||
stopped = true
|
||||
if (timer !== undefined) clearInterval(timer)
|
||||
try {
|
||||
streamController?.close()
|
||||
} catch {
|
||||
// The production reader may already have cancelled the stream.
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function driveWithGuard(
|
||||
response: Response,
|
||||
options: {
|
||||
idleTimeoutMs: number
|
||||
hardMaxQueryMs: number
|
||||
/** Raw-reader deadline, intentionally distinct from QueryGuard activity. */
|
||||
readerIdleTimeoutMs: number
|
||||
},
|
||||
): Promise<{
|
||||
events: AnthropicStreamEvent[]
|
||||
timeout: QueryGuardTimeoutInfo
|
||||
result: { status: 'resolved' } | { status: 'rejected'; error: unknown }
|
||||
signalReason: unknown
|
||||
}> {
|
||||
const controller = new AbortController()
|
||||
const guard = new QueryGuard(options)
|
||||
const start = guard.tryStart({
|
||||
queryId: 'issue-1830-query',
|
||||
querySource: 'issue-1830-test',
|
||||
})
|
||||
if (!start) throw new Error('QueryGuard did not start')
|
||||
let timeout: QueryGuardTimeoutInfo | undefined
|
||||
guard.setTimeoutHandler(info => {
|
||||
timeout = info
|
||||
requestAbort(controller, info.context.terminalReason, {
|
||||
source: 'query_guard',
|
||||
subsystem: 'issue_1830_test',
|
||||
controllerRole: 'query-root',
|
||||
causalEventId: info.causalEventId,
|
||||
})
|
||||
})
|
||||
const events: AnthropicStreamEvent[] = []
|
||||
const stream = __codexStreamToAnthropicForTests(
|
||||
response,
|
||||
'gpt-test',
|
||||
controller.signal,
|
||||
{ idleTimeoutMs: options.readerIdleTimeoutMs },
|
||||
)
|
||||
try {
|
||||
const result = await bounded(
|
||||
driveQueryEvents(
|
||||
stream,
|
||||
reason => guard.registerActivity(reason, start.generation),
|
||||
event => events.push(event),
|
||||
).then(
|
||||
() => ({ status: 'resolved' as const }),
|
||||
error => ({ status: 'rejected' as const, error }),
|
||||
),
|
||||
)
|
||||
if (!timeout) throw new Error('QueryGuard did not own the terminal decision')
|
||||
return { events, timeout, result, signalReason: controller.signal.reason }
|
||||
} finally {
|
||||
if (!controller.signal.aborted) {
|
||||
requestAbort(controller, 'test-cleanup', {
|
||||
source: 'issue_1830_test_cleanup',
|
||||
subsystem: 'issue_1830_test',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
guard.forceEnd('unknown', 'test-cleanup')
|
||||
await bounded(stream.return(undefined), 100).catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
describe('issue #1830 Codex interruption ownership', () => {
|
||||
test('raw transport silence is owned by the Codex reader deadline', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const cancelReasons: unknown[] = []
|
||||
const response = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
cancel(reason) {
|
||||
cancelReasons.push(reason)
|
||||
},
|
||||
}),
|
||||
)
|
||||
const iterator = __codexStreamToAnthropicForTests(
|
||||
response,
|
||||
'gpt-test',
|
||||
undefined,
|
||||
{ idleTimeoutMs: 25 },
|
||||
)[Symbol.asyncIterator]()
|
||||
|
||||
try {
|
||||
expect((await bounded(iterator.next())).value?.type).toBe('message_start')
|
||||
const result = await bounded(
|
||||
iterator.next().then(
|
||||
value => ({ status: 'resolved' as const, value }),
|
||||
error => ({ status: 'rejected' as const, error }),
|
||||
),
|
||||
)
|
||||
|
||||
expect(result.status).toBe('rejected')
|
||||
if (result.status === 'rejected') {
|
||||
expect((result.error as Error).message).toContain(
|
||||
'Codex SSE stream idle',
|
||||
)
|
||||
}
|
||||
expect(cancelReasons).toHaveLength(1)
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const idleTimeout = trace.find(
|
||||
entry => entry.event === 'codex_stream.idle_timeout',
|
||||
)
|
||||
const readerError = trace.find(
|
||||
entry => entry.event === 'codex_stream.error',
|
||||
)
|
||||
const readerCancelled = trace.find(
|
||||
entry => entry.event === 'codex_stream.cancelled',
|
||||
)
|
||||
const converterClosed = trace.find(
|
||||
entry => entry.event === 'codex_stream.converter_closed',
|
||||
)
|
||||
expect(idleTimeout).toBeDefined()
|
||||
expect(readerError).toBeDefined()
|
||||
expect(readerCancelled).toBeDefined()
|
||||
expect(converterClosed).toBeDefined()
|
||||
expect(typeof idleTimeout!.eventId).toBe('string')
|
||||
expect(typeof readerError!.causalEventId).toBe('string')
|
||||
expect(typeof readerCancelled!.causalEventId).toBe('string')
|
||||
expect(typeof converterClosed!.causalEventId).toBe('string')
|
||||
expect(readerError!.causalEventId).toBe(idleTimeout!.eventId)
|
||||
expect(readerCancelled!.causalEventId).toBe(idleTimeout!.eventId)
|
||||
expect(converterClosed!.causalEventId).toBe(idleTimeout!.eventId)
|
||||
} finally {
|
||||
const returned = iterator.return?.(undefined)
|
||||
if (returned) await bounded(Promise.resolve(returned), 100).catch(() => {})
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('parent abort settles the pending read before its idle deadline', async () => {
|
||||
const cancelReasons: unknown[] = []
|
||||
const response = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
cancel(reason) {
|
||||
cancelReasons.push(reason)
|
||||
},
|
||||
}),
|
||||
)
|
||||
const controller = new AbortController()
|
||||
const iterator = __codexStreamToAnthropicForTests(
|
||||
response,
|
||||
'gpt-test',
|
||||
controller.signal,
|
||||
{ idleTimeoutMs: 1_000 },
|
||||
)[Symbol.asyncIterator]()
|
||||
try {
|
||||
expect((await bounded(iterator.next())).value?.type).toBe('message_start')
|
||||
|
||||
const pending = iterator.next().then(
|
||||
value => ({ status: 'resolved' as const, value }),
|
||||
error => ({ status: 'rejected' as const, error }),
|
||||
)
|
||||
controller.abort('query-timeout')
|
||||
const result = await bounded(pending)
|
||||
|
||||
expect(controller.signal.reason).toBe('query-timeout')
|
||||
expect(result.status).toBe('rejected')
|
||||
if (result.status === 'rejected') {
|
||||
expect((result.error as { name?: unknown }).name).toBe('AbortError')
|
||||
}
|
||||
expect(cancelReasons).toHaveLength(1)
|
||||
} finally {
|
||||
if (!controller.signal.aborted) controller.abort('test-cleanup')
|
||||
const returned = iterator.return?.(undefined)
|
||||
if (returned) await bounded(Promise.resolve(returned), 100).catch(() => {})
|
||||
}
|
||||
})
|
||||
|
||||
test('a done-only Codex stream completes without waiting for transport EOF', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const cancelReasons: unknown[] = []
|
||||
const encoder = new TextEncoder()
|
||||
const response = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode('data: [DONE]\n\n'))
|
||||
},
|
||||
cancel(reason) {
|
||||
cancelReasons.push(reason)
|
||||
},
|
||||
}),
|
||||
)
|
||||
|
||||
try {
|
||||
const events: AnthropicStreamEvent[] = []
|
||||
for await (const event of __codexStreamToAnthropicForTests(
|
||||
response,
|
||||
'gpt-test',
|
||||
undefined,
|
||||
{ idleTimeoutMs: 25 },
|
||||
)) {
|
||||
events.push(event)
|
||||
}
|
||||
expect(events.at(-1)?.type).toBe('message_stop')
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().some(
|
||||
entry => entry.event === 'codex_stream.idle_timeout',
|
||||
),
|
||||
).toBe(false)
|
||||
expect(cancelReasons).toHaveLength(1)
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
})
|
||||
|
||||
test('normal completed and incomplete frames after a done marker close without interruption traces', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
try {
|
||||
for (const terminalEvent of [
|
||||
'response.completed',
|
||||
'response.incomplete',
|
||||
]) {
|
||||
__resetInterruptionTraceForTests()
|
||||
const cancelReasons: unknown[] = []
|
||||
const encoder = new TextEncoder()
|
||||
const response = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array())
|
||||
controller.enqueue(encoder.encode([
|
||||
': keepalive',
|
||||
'',
|
||||
'data: [DONE]',
|
||||
'',
|
||||
`event: ${terminalEvent}`,
|
||||
`data: {"type":"${terminalEvent}","response":{"status":"${terminalEvent.slice('response.'.length)}","output":[]}}`,
|
||||
'',
|
||||
'',
|
||||
].join('\n')))
|
||||
},
|
||||
cancel(reason) {
|
||||
cancelReasons.push(reason)
|
||||
},
|
||||
}),
|
||||
)
|
||||
const events: AnthropicStreamEvent[] = []
|
||||
await bounded((async () => {
|
||||
for await (const event of codexStreamToAnthropic(response, 'gpt-test')) {
|
||||
events.push(event)
|
||||
}
|
||||
})())
|
||||
|
||||
expect(events.at(-1)?.type).toBe('message_stop')
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const firstRawBytes = trace.filter(
|
||||
entry => entry.event === 'codex_stream.first_raw_byte',
|
||||
)
|
||||
const terminal = trace.find(
|
||||
entry =>
|
||||
entry.event === 'codex_stream.protocol_terminal' &&
|
||||
entry.phase === terminalEvent,
|
||||
)
|
||||
expect(firstRawBytes).toHaveLength(1)
|
||||
expect(firstRawBytes[0]!.rawByteCount).toBeGreaterThan(0)
|
||||
expect(terminal).toBeDefined()
|
||||
expect(terminal!.controlFrameCount).toBe(2)
|
||||
expect(terminal!.ignoredFrameCount).toBe(0)
|
||||
expect(
|
||||
trace.some(entry => entry.event === 'codex_stream.cancelled'),
|
||||
).toBe(false)
|
||||
expect(cancelReasons).toHaveLength(1)
|
||||
}
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('reports non-negative idle evidence and forwards its causal event id', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const causalEventIds: string[] = []
|
||||
const encoder = new TextEncoder()
|
||||
const response = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(': keepalive\n\n'))
|
||||
},
|
||||
}),
|
||||
)
|
||||
const iterator = __codexStreamToAnthropicForTests(
|
||||
response,
|
||||
'gpt-test',
|
||||
undefined,
|
||||
{
|
||||
idleTimeoutMs: 25,
|
||||
onCausalEventId: eventId => causalEventIds.push(eventId),
|
||||
},
|
||||
)[Symbol.asyncIterator]()
|
||||
|
||||
try {
|
||||
expect((await bounded(iterator.next())).value?.type).toBe('message_start')
|
||||
const pending = iterator.next().catch(error => error)
|
||||
await bounded(pending)
|
||||
|
||||
const idleTimeout = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'codex_stream.idle_timeout',
|
||||
)
|
||||
expect(idleTimeout).toBeDefined()
|
||||
expect(idleTimeout?.sinceLastRawByteMs).toBeGreaterThanOrEqual(0)
|
||||
expect(idleTimeout?.sinceLastParsedFrameMs).toBeGreaterThanOrEqual(0)
|
||||
expect(causalEventIds).toEqual([idleTimeout!.eventId])
|
||||
} finally {
|
||||
await bounded(iterator.return?.(undefined) ?? Promise.resolve()).catch(
|
||||
() => {},
|
||||
)
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
})
|
||||
|
||||
test('marks the converter complete before yielding message_stop', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const cancelReasons: unknown[] = []
|
||||
const encoder = new TextEncoder()
|
||||
const response = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode([
|
||||
'event: response.completed',
|
||||
'data: {"type":"response.completed","response":{"status":"completed","output":[]}}',
|
||||
'',
|
||||
'',
|
||||
].join('\n')))
|
||||
},
|
||||
cancel(reason) {
|
||||
cancelReasons.push(reason)
|
||||
},
|
||||
}),
|
||||
)
|
||||
const iterator = codexStreamToAnthropic(response, 'gpt-test')
|
||||
|
||||
try {
|
||||
let next: IteratorResult<AnthropicStreamEvent>
|
||||
do {
|
||||
next = await bounded(iterator.next())
|
||||
} while (!next.done && next.value.type !== 'message_stop')
|
||||
expect(next.done).toBe(false)
|
||||
|
||||
await bounded(iterator.return(undefined))
|
||||
|
||||
const converterClosed = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'codex_stream.converter_closed',
|
||||
)
|
||||
expect(converterClosed).toBeDefined()
|
||||
expect(converterClosed!.outcome).toBe('complete')
|
||||
expect(cancelReasons).toHaveLength(1)
|
||||
} finally {
|
||||
await bounded(iterator.return(undefined)).catch(() => {})
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('reports root_aborted when cancellation follows terminal evidence', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const controller = new AbortController()
|
||||
const response = responseFromText([
|
||||
'event: response.completed',
|
||||
'data: {"type":"response.completed","response":{"status":"completed","output":[]}}',
|
||||
'',
|
||||
'',
|
||||
].join('\n'))
|
||||
const iterator = codexStreamToAnthropic(
|
||||
response,
|
||||
'gpt-test',
|
||||
controller.signal,
|
||||
)
|
||||
|
||||
let next: IteratorResult<AnthropicStreamEvent>
|
||||
do {
|
||||
next = await bounded(iterator.next())
|
||||
} while (!next.done && next.value.type !== 'message_stop')
|
||||
expect(next.done).toBe(false)
|
||||
|
||||
requestAbort(controller, undefined, {
|
||||
source: 'test_root_abort',
|
||||
subsystem: 'query_engine',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
await bounded(iterator.return(undefined))
|
||||
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'codex_stream.converter_closed',
|
||||
)?.outcome,
|
||||
).toBe('root_aborted')
|
||||
})
|
||||
|
||||
test('keeps terminal ownership when the consumer returns after message_delta', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const response = responseFromText([
|
||||
'event: response.completed',
|
||||
'data: {"type":"response.completed","response":{"status":"completed","output":[]}}',
|
||||
'',
|
||||
'',
|
||||
].join('\n'))
|
||||
const iterator = codexStreamToAnthropic(response, 'gpt-test')
|
||||
|
||||
try {
|
||||
let next: IteratorResult<AnthropicStreamEvent>
|
||||
do {
|
||||
next = await bounded(iterator.next())
|
||||
} while (!next.done && next.value.type !== 'message_delta')
|
||||
expect(next.done).toBe(false)
|
||||
await bounded(iterator.return(undefined))
|
||||
|
||||
const converterClosed = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'codex_stream.converter_closed',
|
||||
)
|
||||
expect(converterClosed?.outcome).toBe('complete')
|
||||
} finally {
|
||||
await bounded(iterator.return(undefined)).catch(() => {})
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
})
|
||||
|
||||
test('classifies each Codex reader and converter frame exactly once', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const text = [
|
||||
': keepalive',
|
||||
'',
|
||||
'event: response.created',
|
||||
'data: not-json',
|
||||
'',
|
||||
'event: response.created',
|
||||
'data: []',
|
||||
'',
|
||||
'event: response.created',
|
||||
'data: {"type":"response.created","sequence_number":1}',
|
||||
'',
|
||||
'data: [DONE]',
|
||||
'',
|
||||
'event: response.completed',
|
||||
'data: {"type":"response.completed","response":{"status":"completed","output":[]}}',
|
||||
'',
|
||||
'',
|
||||
].join('\n')
|
||||
|
||||
try {
|
||||
await bounded((async () => {
|
||||
for await (const _event of codexStreamToAnthropic(
|
||||
responseFromText(text),
|
||||
'gpt-test',
|
||||
)) {
|
||||
// Drain the converter to terminal diagnostics.
|
||||
}
|
||||
})())
|
||||
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const terminal = trace.find(
|
||||
entry => entry.event === 'codex_stream.protocol_terminal',
|
||||
)
|
||||
const converterClosed = trace.find(
|
||||
entry => entry.event === 'codex_stream.converter_closed',
|
||||
)
|
||||
expect(terminal).toMatchObject({
|
||||
rawByteCount: new TextEncoder().encode(text).byteLength,
|
||||
parsedFrameCount: 2,
|
||||
controlFrameCount: 2,
|
||||
ignoredFrameCount: 2,
|
||||
})
|
||||
expect(converterClosed?.ignoredParsedFrameCount).toBe(1)
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
})
|
||||
|
||||
test('classifies event-only and typed data-only frames without inflating ignored input', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const text = [
|
||||
'event: ping',
|
||||
'',
|
||||
'data: {"type":"response.created","sequence_number":1}',
|
||||
'',
|
||||
'event: response.completed',
|
||||
'data: {"type":"response.completed","response":{"status":"completed","output":[]}}',
|
||||
'',
|
||||
'',
|
||||
].join('\n')
|
||||
|
||||
try {
|
||||
await collectCodex(responseFromText(text))
|
||||
const terminal = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'codex_stream.protocol_terminal',
|
||||
)
|
||||
expect(terminal).toMatchObject({
|
||||
parsedFrameCount: 2,
|
||||
controlFrameCount: 1,
|
||||
ignoredFrameCount: 0,
|
||||
})
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
})
|
||||
|
||||
test('reports parsed-but-unhandled converter events with a distinct counter', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const encoder = new TextEncoder()
|
||||
const response = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode([
|
||||
'event: response.created',
|
||||
'data: {"type":"response.created","sequence_number":1}',
|
||||
'',
|
||||
'event: response.completed',
|
||||
'data: {"type":"response.completed","response":{"status":"completed","output":[]}}',
|
||||
'',
|
||||
'',
|
||||
].join('\n')))
|
||||
},
|
||||
}),
|
||||
)
|
||||
|
||||
try {
|
||||
await bounded((async () => {
|
||||
for await (const _event of codexStreamToAnthropic(response, 'gpt-test')) {
|
||||
// Drain the converter to its terminal diagnostics.
|
||||
}
|
||||
})())
|
||||
|
||||
const converterClosed = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'codex_stream.converter_closed',
|
||||
)
|
||||
expect(converterClosed?.ignoredParsedFrameCount).toBe(1)
|
||||
expect(converterClosed?.ignoredFrameCount).toBeUndefined()
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
})
|
||||
|
||||
test('keepalives and parsed-but-ignored frames cannot reset QueryGuard', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const timed = makeTimedStream(
|
||||
index => {
|
||||
if (index % 4 === 0) return ': keepalive\n\n'
|
||||
if (index % 4 === 1) {
|
||||
return `event: response.created\ndata: {"type":"response.created","sequence_number":${index}}\n\n`
|
||||
}
|
||||
if (index % 4 === 2) {
|
||||
return 'event: response.created\ndata: []\n\n'
|
||||
}
|
||||
return `event: response.reasoning_summary_text.delta\ndata: {"type":"response.reasoning_summary_text.delta","delta":"r","sequence_number":${index}}\n\n`
|
||||
},
|
||||
25,
|
||||
)
|
||||
try {
|
||||
const outcome = await driveWithGuard(timed.response, {
|
||||
idleTimeoutMs: 250,
|
||||
hardMaxQueryMs: 1_500,
|
||||
readerIdleTimeoutMs: 500,
|
||||
})
|
||||
|
||||
expect(outcome.timeout.reason satisfies QueryGuardTimeoutReason).toBe(
|
||||
'idle',
|
||||
)
|
||||
expect(outcome.signalReason).toBe('query-timeout')
|
||||
expect(outcome.result.status).toBe('rejected')
|
||||
expect(outcome.events.map(event => event.type)).toEqual([
|
||||
'message_start',
|
||||
])
|
||||
const readerClosed = __getInterruptionTraceSnapshotForTests()
|
||||
.filter(entry => entry.event === 'codex_stream.cancelled')
|
||||
.at(-1)
|
||||
const rootAbort = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' && entry.source === 'query_guard',
|
||||
)
|
||||
const traceEvents = __getInterruptionTraceSnapshotForTests().map(
|
||||
entry => entry.event,
|
||||
)
|
||||
expect(timed.getEmissionCount()).toBeGreaterThan(3)
|
||||
expect(readerClosed?.rawByteCount).toBeGreaterThan(0)
|
||||
expect(readerClosed?.parsedFrameCount).toBeGreaterThan(0)
|
||||
expect(readerClosed?.controlFrameCount).toBeGreaterThan(0)
|
||||
expect(readerClosed?.ignoredFrameCount).toBeGreaterThan(0)
|
||||
const converterClosed = __getInterruptionTraceSnapshotForTests()
|
||||
.filter(entry => entry.event === 'codex_stream.converter_closed')
|
||||
.at(-1)
|
||||
expect(converterClosed?.ignoredParsedFrameCount).toBeGreaterThan(0)
|
||||
expect(converterClosed?.ignoredFrameCount).toBeUndefined()
|
||||
expect(rootAbort).toBeDefined()
|
||||
expect(readerClosed).toBeDefined()
|
||||
expect(typeof rootAbort!.eventId).toBe('string')
|
||||
expect(typeof readerClosed!.causalEventId).toBe('string')
|
||||
expect(readerClosed!.causalEventId).toBe(rootAbort!.eventId)
|
||||
expect(traceEvents.indexOf('codex_stream.converter_closed')).toBeGreaterThan(
|
||||
traceEvents.indexOf('codex_stream.cancelled'),
|
||||
)
|
||||
expect(timed.cancelReasons).toHaveLength(1)
|
||||
} finally {
|
||||
timed.stop()
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('valid deltas extend idle activity but cannot bypass the hard maximum', async () => {
|
||||
const timed = makeTimedStream(
|
||||
index =>
|
||||
`event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"x","sequence_number":${index}}\n\n`,
|
||||
25,
|
||||
)
|
||||
try {
|
||||
const outcome = await driveWithGuard(timed.response, {
|
||||
idleTimeoutMs: 250,
|
||||
hardMaxQueryMs: 600,
|
||||
readerIdleTimeoutMs: 500,
|
||||
})
|
||||
const countAtAbort = outcome.events.length
|
||||
await Bun.sleep(30)
|
||||
|
||||
expect(outcome.timeout.reason satisfies QueryGuardTimeoutReason).toBe(
|
||||
'hard_max',
|
||||
)
|
||||
expect(outcome.signalReason).toBe('hard-max-query-timeout')
|
||||
expect(outcome.result.status).toBe('rejected')
|
||||
expect(countAtAbort).toBeGreaterThan(3)
|
||||
expect(outcome.events).toHaveLength(countAtAbort)
|
||||
expect(timed.cancelReasons).toHaveLength(1)
|
||||
} finally {
|
||||
timed.stop()
|
||||
}
|
||||
})
|
||||
|
||||
test('repeated pending-read abort cycles cancel exactly once', async () => {
|
||||
for (let cycle = 0; cycle < 40; cycle++) {
|
||||
const cancelReasons: unknown[] = []
|
||||
const response = new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
cancel(reason) {
|
||||
cancelReasons.push(reason)
|
||||
},
|
||||
}),
|
||||
)
|
||||
const controller = new AbortController()
|
||||
const iterator = __codexStreamToAnthropicForTests(
|
||||
response,
|
||||
'gpt-test',
|
||||
controller.signal,
|
||||
{ idleTimeoutMs: 1_000 },
|
||||
)[Symbol.asyncIterator]()
|
||||
try {
|
||||
expect((await bounded(iterator.next())).value?.type).toBe(
|
||||
'message_start',
|
||||
)
|
||||
const pending = iterator.next().catch(error => error as unknown)
|
||||
controller.abort('user-cancel')
|
||||
await bounded(pending)
|
||||
expect(cancelReasons).toHaveLength(1)
|
||||
} finally {
|
||||
if (!controller.signal.aborted) controller.abort('test-cleanup')
|
||||
const returned = iterator.return?.(undefined)
|
||||
if (returned) {
|
||||
await bounded(Promise.resolve(returned), 100).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
+266
-100
@@ -12,6 +12,18 @@ import {
|
||||
createThinkTagFilter,
|
||||
stripThinkTags,
|
||||
} from './thinkTagSanitizer.js'
|
||||
import {
|
||||
createReaderCanceller,
|
||||
createStreamAbortError,
|
||||
readWithIdleTimeout,
|
||||
throwIfStreamAborted,
|
||||
} from './openaiShim/streamControl.js'
|
||||
import {
|
||||
flushInterruptionTrace,
|
||||
getInterruptionSignalAbortEventId,
|
||||
setInterruptionErrorCausalEventId,
|
||||
traceInterruptionEvent,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
|
||||
export interface AnthropicUsage {
|
||||
input_tokens: number
|
||||
@@ -81,42 +93,6 @@ type CodexSseEvent = {
|
||||
data: Record<string, any>
|
||||
}
|
||||
|
||||
function createStreamAbortError(): DOMException {
|
||||
return new DOMException('Aborted', 'AbortError')
|
||||
}
|
||||
|
||||
function throwIfStreamAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) {
|
||||
throw createStreamAbortError()
|
||||
}
|
||||
}
|
||||
|
||||
function createReaderCanceller(
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
signal?: AbortSignal,
|
||||
): {
|
||||
cancel: (error?: unknown) => void
|
||||
cleanup: () => void
|
||||
} {
|
||||
let cancelled = false
|
||||
const cancel = (error: unknown = createStreamAbortError()) => {
|
||||
if (cancelled) return
|
||||
cancelled = true
|
||||
void reader.cancel(error).catch(() => {})
|
||||
}
|
||||
const onAbort = () => cancel(createStreamAbortError())
|
||||
|
||||
signal?.addEventListener('abort', onAbort, { once: true })
|
||||
if (signal?.aborted) {
|
||||
onAbort()
|
||||
}
|
||||
|
||||
return {
|
||||
cancel,
|
||||
cleanup: () => signal?.removeEventListener('abort', onAbort),
|
||||
}
|
||||
}
|
||||
|
||||
function makeUsage(usage?: Record<string, unknown>): AnthropicUsage {
|
||||
// Single source of truth for raw → Anthropic shape. Lives in
|
||||
// cacheMetrics.ts alongside the raw-shape extractor so any new
|
||||
@@ -708,83 +684,108 @@ export async function performCodexRequest(options: {
|
||||
return response
|
||||
}
|
||||
|
||||
async function* readSseEvents(response: Response, signal?: AbortSignal): AsyncGenerator<CodexSseEvent> {
|
||||
type CodexStreamReadOptions = {
|
||||
/** Internal deterministic-test seam; production keeps the 120-second owner. */
|
||||
idleTimeoutMs?: number
|
||||
onCausalEventId?: (eventId: string) => void
|
||||
}
|
||||
|
||||
const STREAM_IDLE_TIMEOUT_MS = 120_000
|
||||
|
||||
async function* readSseEvents(
|
||||
response: Response,
|
||||
signal?: AbortSignal,
|
||||
options: CodexStreamReadOptions = {},
|
||||
): AsyncGenerator<CodexSseEvent> {
|
||||
const reader = response.body?.getReader()
|
||||
if (!reader) return
|
||||
const readerCanceller = createReaderCanceller(reader, signal)
|
||||
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ''
|
||||
const STREAM_IDLE_TIMEOUT_MS = 120_000 // 2 minutes without data
|
||||
let lastDataTime = Date.now()
|
||||
let streamComplete = false
|
||||
const streamIdleTimeoutMs = options.idleTimeoutMs ?? STREAM_IDLE_TIMEOUT_MS
|
||||
let lastDataTime = performance.now()
|
||||
let lastParsedFrameTime = lastDataTime
|
||||
let transportComplete = false
|
||||
let protocolComplete = false
|
||||
let doneMarkerObserved = false
|
||||
let rawByteCount = 0
|
||||
let sawFirstRawByte = false
|
||||
let parsedFrameCount = 0
|
||||
let controlFrameCount = 0
|
||||
let ignoredFrameCount = 0
|
||||
let streamCausalEventId: string | undefined
|
||||
|
||||
traceInterruptionEvent('codex_stream.read_started', {
|
||||
subsystem: 'codex_stream',
|
||||
transport: 'codex_responses',
|
||||
})
|
||||
|
||||
/**
|
||||
* Read from the stream with an idle timeout. Respects the caller's
|
||||
* AbortSignal — clears the idle timer on abort so the AbortError
|
||||
* surfaces cleanly instead of a spurious idle timeout.
|
||||
*/
|
||||
async function readWithTimeout(): Promise<Bun.ReadableStreamDefaultReadResult<Uint8Array<ArrayBuffer>>> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined = setTimeout(() => {
|
||||
const elapsed = Math.round((Date.now() - lastDataTime) / 1000)
|
||||
cancelAndReject(new Error(
|
||||
`Codex SSE stream idle for ${elapsed}s (limit: ${STREAM_IDLE_TIMEOUT_MS / 1000}s). Connection likely dropped.`,
|
||||
))
|
||||
}, STREAM_IDLE_TIMEOUT_MS)
|
||||
|
||||
const cleanup = () => {
|
||||
if (timeoutId !== undefined) {
|
||||
clearTimeout(timeoutId)
|
||||
timeoutId = undefined
|
||||
}
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
}
|
||||
const finishResolve = (
|
||||
value: Bun.ReadableStreamDefaultReadResult<Uint8Array<ArrayBuffer>>,
|
||||
) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
cleanup()
|
||||
if (value.value) lastDataTime = Date.now()
|
||||
resolve(value)
|
||||
}
|
||||
const finishReject = (error: unknown) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
cleanup()
|
||||
reject(error)
|
||||
}
|
||||
const cancelAndReject = (error: unknown) => {
|
||||
readerCanceller.cancel(error)
|
||||
finishReject(error)
|
||||
}
|
||||
const onAbort = () => cancelAndReject(createStreamAbortError())
|
||||
|
||||
signal?.addEventListener('abort', onAbort, { once: true })
|
||||
if (signal?.aborted) {
|
||||
onAbort()
|
||||
return
|
||||
}
|
||||
|
||||
// reader is guarded non-null above; hoisted function escapes TS narrowing.
|
||||
reader!.read().then(
|
||||
result => finishResolve(result),
|
||||
err => finishReject(err),
|
||||
async function readWithTimeout() {
|
||||
return readWithIdleTimeout(reader!, streamIdleTimeoutMs, {
|
||||
signal,
|
||||
cancelReader: readerCanceller.cancel,
|
||||
createTimeoutError: () => {
|
||||
const elapsed = Math.round((performance.now() - lastDataTime) / 1000)
|
||||
return new Error(
|
||||
`Codex SSE stream idle for ${elapsed}s (limit: ${streamIdleTimeoutMs / 1000}s). Connection likely dropped.`,
|
||||
)
|
||||
},
|
||||
onTimeout: error => {
|
||||
const now = performance.now()
|
||||
streamCausalEventId = traceInterruptionEvent('codex_stream.idle_timeout', {
|
||||
subsystem: 'codex_stream',
|
||||
transport: 'codex_responses',
|
||||
sinceLastRawByteMs: now - lastDataTime,
|
||||
sinceLastParsedFrameMs: now - lastParsedFrameTime,
|
||||
rawByteCount,
|
||||
parsedFrameCount,
|
||||
controlFrameCount,
|
||||
ignoredFrameCount,
|
||||
})
|
||||
if (streamCausalEventId) {
|
||||
options.onCausalEventId?.(streamCausalEventId)
|
||||
}
|
||||
setInterruptionErrorCausalEventId(error, streamCausalEventId)
|
||||
flushInterruptionTrace('codex_stream_idle_timeout')
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
streamLoop: while (true) {
|
||||
const { done, value } = await readWithTimeout()
|
||||
if (done) {
|
||||
streamComplete = true
|
||||
transportComplete = true
|
||||
traceInterruptionEvent('codex_stream.eof', {
|
||||
subsystem: 'codex_stream',
|
||||
transport: 'codex_responses',
|
||||
rawByteCount,
|
||||
parsedFrameCount,
|
||||
controlFrameCount,
|
||||
ignoredFrameCount,
|
||||
})
|
||||
break
|
||||
}
|
||||
|
||||
throwIfStreamAborted(signal)
|
||||
if (value && value.byteLength > 0) {
|
||||
lastDataTime = performance.now()
|
||||
rawByteCount += value.byteLength
|
||||
if (!sawFirstRawByte) {
|
||||
sawFirstRawByte = true
|
||||
traceInterruptionEvent('codex_stream.first_raw_byte', {
|
||||
subsystem: 'codex_stream',
|
||||
transport: 'codex_responses',
|
||||
rawByteCount,
|
||||
})
|
||||
}
|
||||
}
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
const chunks = buffer.split('\n\n')
|
||||
buffer = chunks.pop() ?? ''
|
||||
@@ -797,33 +798,132 @@ async function* readSseEvents(response: Response, signal?: AbortSignal): AsyncGe
|
||||
.filter(Boolean)
|
||||
if (lines.length === 0) continue
|
||||
|
||||
if (lines.every(line => line.startsWith(':'))) {
|
||||
controlFrameCount++
|
||||
continue
|
||||
}
|
||||
|
||||
const eventLine = lines.find(line => line.startsWith('event: '))
|
||||
const dataLines = lines.filter(line => line.startsWith('data: '))
|
||||
if (!eventLine || dataLines.length === 0) continue
|
||||
if (dataLines.length === 0) {
|
||||
controlFrameCount++
|
||||
continue
|
||||
}
|
||||
|
||||
const event = eventLine.slice(7).trim()
|
||||
const rawData = dataLines.map(line => line.slice(6)).join('\n')
|
||||
if (rawData === '[DONE]') continue
|
||||
if (rawData === '[DONE]') {
|
||||
doneMarkerObserved = true
|
||||
controlFrameCount++
|
||||
traceInterruptionEvent('codex_stream.control_frame', {
|
||||
subsystem: 'codex_stream',
|
||||
transport: 'codex_responses',
|
||||
phase: 'done_marker',
|
||||
rawByteCount,
|
||||
parsedFrameCount,
|
||||
controlFrameCount,
|
||||
ignoredFrameCount,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
let data: Record<string, any>
|
||||
try {
|
||||
const parsed = JSON.parse(rawData)
|
||||
if (!parsed || typeof parsed !== 'object') continue
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
ignoredFrameCount++
|
||||
continue
|
||||
}
|
||||
data = parsed as Record<string, any>
|
||||
} catch {
|
||||
ignoredFrameCount++
|
||||
continue
|
||||
}
|
||||
|
||||
const event = eventLine?.slice(7).trim() ??
|
||||
(typeof data.type === 'string' ? data.type : undefined)
|
||||
if (!event) {
|
||||
controlFrameCount++
|
||||
continue
|
||||
}
|
||||
|
||||
throwIfStreamAborted(signal)
|
||||
parsedFrameCount++
|
||||
lastParsedFrameTime = performance.now()
|
||||
if (parsedFrameCount === 1) {
|
||||
traceInterruptionEvent('codex_stream.first_parsed_frame', {
|
||||
subsystem: 'codex_stream',
|
||||
transport: 'codex_responses',
|
||||
rawByteCount,
|
||||
parsedFrameCount,
|
||||
controlFrameCount,
|
||||
ignoredFrameCount,
|
||||
})
|
||||
}
|
||||
const isTerminalEvent =
|
||||
event === 'response.completed' ||
|
||||
event === 'response.incomplete' ||
|
||||
event === 'response.failed'
|
||||
if (isTerminalEvent) {
|
||||
protocolComplete = true
|
||||
traceInterruptionEvent('codex_stream.protocol_terminal', {
|
||||
subsystem: 'codex_stream',
|
||||
transport: 'codex_responses',
|
||||
phase: event,
|
||||
rawByteCount,
|
||||
parsedFrameCount,
|
||||
controlFrameCount,
|
||||
ignoredFrameCount,
|
||||
})
|
||||
}
|
||||
yield { event, data }
|
||||
if (isTerminalEvent) break streamLoop
|
||||
}
|
||||
if (doneMarkerObserved) {
|
||||
protocolComplete = true
|
||||
break
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
traceInterruptionEvent('codex_stream.error', {
|
||||
subsystem: 'codex_stream',
|
||||
transport: 'codex_responses',
|
||||
outcome: signal?.aborted ? 'root_aborted' : 'external_error',
|
||||
reason: signal?.reason,
|
||||
causalEventId:
|
||||
(signal && getInterruptionSignalAbortEventId(signal)) ??
|
||||
streamCausalEventId,
|
||||
error,
|
||||
rawByteCount,
|
||||
parsedFrameCount,
|
||||
controlFrameCount,
|
||||
ignoredFrameCount,
|
||||
})
|
||||
flushInterruptionTrace('codex_stream_error')
|
||||
throw error
|
||||
} finally {
|
||||
if (!streamComplete || signal?.aborted) {
|
||||
readerCanceller.cancel(createStreamAbortError())
|
||||
const readerWasInterrupted =
|
||||
(!transportComplete && !protocolComplete) || signal?.aborted
|
||||
if (readerWasInterrupted) {
|
||||
const causalEventId =
|
||||
(signal && getInterruptionSignalAbortEventId(signal)) ??
|
||||
streamCausalEventId
|
||||
traceInterruptionEvent('codex_stream.cancelled', {
|
||||
subsystem: 'codex_stream',
|
||||
transport: 'codex_responses',
|
||||
reason: signal?.reason,
|
||||
causalEventId,
|
||||
rawByteCount,
|
||||
parsedFrameCount,
|
||||
controlFrameCount,
|
||||
ignoredFrameCount,
|
||||
})
|
||||
}
|
||||
if (!transportComplete) readerCanceller.cancel(createStreamAbortError())
|
||||
readerCanceller.cleanup()
|
||||
reader.releaseLock()
|
||||
if (readerWasInterrupted) {
|
||||
flushInterruptionTrace('codex_stream_reader_closed')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -882,10 +982,11 @@ export async function collectCodexCompletedResponse(
|
||||
return completedResponse
|
||||
}
|
||||
|
||||
export async function* codexStreamToAnthropic(
|
||||
async function* codexStreamToAnthropicWithReadOptions(
|
||||
response: Response,
|
||||
model: string,
|
||||
signal?: AbortSignal,
|
||||
readOptions: CodexStreamReadOptions = {},
|
||||
): AsyncGenerator<AnthropicStreamEvent> {
|
||||
const messageId = makeMessageId()
|
||||
const toolBlocksByItemId = new Map<
|
||||
@@ -898,6 +999,10 @@ export async function* codexStreamToAnthropic(
|
||||
let sawToolUse = false
|
||||
let finalResponse: Record<string, any> | undefined
|
||||
let streamComplete = false
|
||||
let providerTerminalObserved = false
|
||||
let converterFailed = false
|
||||
let streamCausalEventId: string | undefined
|
||||
let ignoredParsedFrameCount = 0
|
||||
const cancelResponseBody = () => {
|
||||
void response.body?.cancel(createStreamAbortError()).catch(() => {})
|
||||
}
|
||||
@@ -953,7 +1058,13 @@ export async function* codexStreamToAnthropic(
|
||||
},
|
||||
}
|
||||
|
||||
for await (const event of readSseEvents(response, signal)) {
|
||||
for await (const event of readSseEvents(response, signal, {
|
||||
...readOptions,
|
||||
onCausalEventId: eventId => {
|
||||
streamCausalEventId = eventId
|
||||
readOptions.onCausalEventId?.(eventId)
|
||||
},
|
||||
})) {
|
||||
throwIfStreamAborted(signal)
|
||||
const payload = event.data
|
||||
|
||||
@@ -1113,15 +1224,26 @@ export async function* codexStreamToAnthropic(
|
||||
event.event === 'response.completed' ||
|
||||
event.event === 'response.incomplete'
|
||||
) {
|
||||
providerTerminalObserved = true
|
||||
finalResponse = payload.response
|
||||
break
|
||||
}
|
||||
|
||||
if (event.event === 'response.failed') {
|
||||
providerTerminalObserved = true
|
||||
const msg = payload?.response?.error?.message ??
|
||||
payload?.error?.message ?? 'Codex response failed'
|
||||
throw APIError.generate(500, undefined, msg, new Headers())
|
||||
}
|
||||
|
||||
ignoredParsedFrameCount++
|
||||
if (ignoredParsedFrameCount === 1) {
|
||||
traceInterruptionEvent('codex_stream.first_parsed_frame_ignored', {
|
||||
subsystem: 'codex_stream',
|
||||
transport: 'codex_responses',
|
||||
ignoredParsedFrameCount,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
throwIfStreamAborted(signal)
|
||||
@@ -1151,16 +1273,60 @@ export async function* codexStreamToAnthropic(
|
||||
),
|
||||
}
|
||||
throwIfStreamAborted(signal)
|
||||
yield { type: 'message_stop' }
|
||||
streamComplete = true
|
||||
yield { type: 'message_stop' }
|
||||
} catch (error) {
|
||||
converterFailed = true
|
||||
throw error
|
||||
} finally {
|
||||
if (!streamComplete || signal?.aborted) {
|
||||
const terminalComplete = providerTerminalObserved && !converterFailed
|
||||
traceInterruptionEvent('codex_stream.converter_closed', {
|
||||
subsystem: 'codex_stream',
|
||||
transport: 'codex_responses',
|
||||
outcome: signal?.aborted
|
||||
? 'root_aborted'
|
||||
: converterFailed
|
||||
? 'failed'
|
||||
: streamComplete || terminalComplete
|
||||
? 'complete'
|
||||
: 'incomplete',
|
||||
reason: signal?.reason,
|
||||
causalEventId:
|
||||
(signal && getInterruptionSignalAbortEventId(signal)) ??
|
||||
streamCausalEventId,
|
||||
ignoredParsedFrameCount,
|
||||
})
|
||||
if ((!streamComplete && !terminalComplete) || signal?.aborted) {
|
||||
cancelResponseBody()
|
||||
flushInterruptionTrace('codex_stream_converter_closed')
|
||||
}
|
||||
signal?.removeEventListener('abort', cancelResponseBody)
|
||||
}
|
||||
}
|
||||
|
||||
export function codexStreamToAnthropic(
|
||||
response: Response,
|
||||
model: string,
|
||||
signal?: AbortSignal,
|
||||
): AsyncGenerator<AnthropicStreamEvent> {
|
||||
return codexStreamToAnthropicWithReadOptions(response, model, signal)
|
||||
}
|
||||
|
||||
/** Deterministic reader-deadline seam for interruption regressions only. */
|
||||
export function __codexStreamToAnthropicForTests(
|
||||
response: Response,
|
||||
model: string,
|
||||
signal: AbortSignal | undefined,
|
||||
readOptions: CodexStreamReadOptions,
|
||||
): AsyncGenerator<AnthropicStreamEvent> {
|
||||
return codexStreamToAnthropicWithReadOptions(
|
||||
response,
|
||||
model,
|
||||
signal,
|
||||
readOptions,
|
||||
)
|
||||
}
|
||||
|
||||
export function convertCodexResponseToAnthropicMessage(
|
||||
data: Record<string, any>,
|
||||
model: string,
|
||||
|
||||
@@ -6,6 +6,10 @@ import { asMockFetch } from '../../test/typedMocks.js'
|
||||
import { _clearRegistryForTesting, ensureIntegrationsLoaded, registerGateway } from '../../integrations/index.ts'
|
||||
import { applyProviderFlag } from '../../utils/providerFlag.ts'
|
||||
import { applyProviderProfileToProcessEnv } from '../../utils/providerProfiles.ts'
|
||||
import {
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
import {
|
||||
getAssistantMessageFromError,
|
||||
OPENCODE_GO_FREE_LIMIT_ERROR_MESSAGE,
|
||||
@@ -4751,6 +4755,49 @@ test('caller abort winning the timeout catch race prevents a retry', async () =>
|
||||
expect(fetchCalls).toBe(1)
|
||||
})
|
||||
|
||||
test('interruption tracing preserves the native AbortSignal.any request path', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
const originalAbortSignalAny = Object.getOwnPropertyDescriptor(
|
||||
AbortSignal,
|
||||
'any',
|
||||
)
|
||||
const nativeAny = AbortSignal.any.bind(AbortSignal)
|
||||
let nativeAnyCalls = 0
|
||||
Object.defineProperty(AbortSignal, 'any', {
|
||||
configurable: true,
|
||||
value: (signals: AbortSignal[]) => {
|
||||
nativeAnyCalls++
|
||||
return nativeAny(signals)
|
||||
},
|
||||
})
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
globalThis.fetch = asMockFetch(
|
||||
mock(async () => makeChatCompletionResponse('gpt-4o-mini')),
|
||||
)
|
||||
|
||||
try {
|
||||
const client = createOpenAIShimClient({}) as OpenAIShimClient
|
||||
await client.beta.messages.create(
|
||||
{
|
||||
model: 'gpt-4o-mini',
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
max_tokens: 64,
|
||||
stream: false,
|
||||
},
|
||||
{ signal: new AbortController().signal },
|
||||
)
|
||||
expect(nativeAnyCalls).toBe(1)
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
if (originalAbortSignalAny) {
|
||||
Object.defineProperty(AbortSignal, 'any', originalAbortSignalAny)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('manual signal fallback preserves caller cancellation after headers arrive', async () => {
|
||||
process.env.API_TIMEOUT_MS = '200'
|
||||
const fetchSignals: AbortSignal[] = []
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
import { expect, test } from 'bun:test'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../../../test/sharedMutationLock.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
} from '../../../utils/interruptionTrace.js'
|
||||
import type { AnthropicStreamEvent, ShimCreateParams } from '../codexShim.js'
|
||||
import {
|
||||
createShimRequest,
|
||||
@@ -107,6 +116,47 @@ test('OpenAIShimStream combines parent and controller cancellation', async () =>
|
||||
expect(receivedSignal?.aborted).toBe(true)
|
||||
})
|
||||
|
||||
test('OpenAIShimStream records its parent signal relationship', async () => {
|
||||
await acquireSharedMutationLock('openaiShim-clientDispatch-parent-trace')
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const parent = new AbortController()
|
||||
let stream: OpenAIShimStream | undefined
|
||||
try {
|
||||
stream = new OpenAIShimStream(async function* () {
|
||||
yield { type: 'unused' }
|
||||
}, parent.signal)
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const parentRegistration = trace.find(
|
||||
entry =>
|
||||
entry.event === 'signal.registered' &&
|
||||
entry.controllerRole === 'combined-parent',
|
||||
)
|
||||
const streamRegistration = trace.find(
|
||||
entry =>
|
||||
entry.event === 'controller.registered' &&
|
||||
entry.controllerRole === 'stream-controller',
|
||||
)
|
||||
expect(parentRegistration).toBeDefined()
|
||||
expect(streamRegistration).toBeDefined()
|
||||
if (!parentRegistration?.controllerId || !streamRegistration) {
|
||||
throw new Error('missing interruption controller registration')
|
||||
}
|
||||
expect(streamRegistration.parentControllerIds).toEqual([
|
||||
parentRegistration.controllerId,
|
||||
])
|
||||
} finally {
|
||||
stream?.controller.abort('test-cleanup')
|
||||
parent.abort('test-cleanup')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
test('OpenAIShimStream cancels the response before iteration starts', () => {
|
||||
let cancellations = 0
|
||||
const stream = new OpenAIShimStream(
|
||||
@@ -123,12 +173,61 @@ test('OpenAIShimStream cancels the response before iteration starts', () => {
|
||||
})
|
||||
|
||||
test('OpenAIShimStream aborts its controller when a consumer returns early', async () => {
|
||||
await acquireSharedMutationLock('openaiShim-clientDispatch-closure-trace')
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const stream = new OpenAIShimStream(async function* () {
|
||||
yield { type: 'first' }
|
||||
yield { type: 'second' }
|
||||
})
|
||||
try {
|
||||
for await (const _event of stream) break
|
||||
expect(stream.controller.signal.aborted).toBe(true)
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'abort.requested',
|
||||
),
|
||||
).toMatchObject({ source: 'iterator_closed' })
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
test('OpenAIShimStream does not relabel a provider exception as consumer closure', async () => {
|
||||
await acquireSharedMutationLock('openaiShim-clientDispatch-failure-trace')
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const stream = new OpenAIShimStream(async function* () {
|
||||
yield { type: 'message_start' }
|
||||
throw new Error('provider generator failed')
|
||||
})
|
||||
|
||||
try {
|
||||
const iterator = stream[Symbol.asyncIterator]()
|
||||
expect((await iterator.next()).done).toBe(false)
|
||||
await expect(iterator.next()).rejects.toThrow('provider generator failed')
|
||||
expect(stream.controller.signal.aborted).toBe(false)
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().some(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.source === 'iterator_closed',
|
||||
),
|
||||
).toBe(false)
|
||||
} finally {
|
||||
stream.controller.abort('test-cleanup')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
test.each([
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import { APIError } from '@anthropic-ai/sdk'
|
||||
import { createCombinedAbortSignal } from '../../../utils/combinedAbortSignal.js'
|
||||
import {
|
||||
registerInterruptionController,
|
||||
registerInterruptionSignal,
|
||||
requestAbort,
|
||||
} from '../../../utils/interruptionTrace.js'
|
||||
import type { AnthropicStreamEvent, ShimCreateParams } from '../codexShim.js'
|
||||
import {
|
||||
isLikelyOllamaEndpoint,
|
||||
@@ -90,6 +95,17 @@ export class OpenAIShimStream implements AsyncIterable<AnthropicStreamEvent> {
|
||||
) {
|
||||
this.makeGenerator = makeGenerator
|
||||
this.parentSignal = parentSignal
|
||||
const parentId = parentSignal
|
||||
? registerInterruptionSignal(parentSignal, {
|
||||
subsystem: 'openai_shim_dispatch',
|
||||
controllerRole: 'combined-parent',
|
||||
})
|
||||
: undefined
|
||||
registerInterruptionController(this.controller, {
|
||||
subsystem: 'openai_shim_dispatch',
|
||||
controllerRole: 'stream-controller',
|
||||
...(parentId && { parentControllerIds: [parentId] }),
|
||||
})
|
||||
|
||||
if (cancelBeforeIteration) {
|
||||
let cleaned = false
|
||||
@@ -129,6 +145,10 @@ export class OpenAIShimStream implements AsyncIterable<AnthropicStreamEvent> {
|
||||
|
||||
const combined = createCombinedAbortSignal(this.parentSignal, {
|
||||
signalB: this.controller.signal,
|
||||
trace: {
|
||||
subsystem: 'openai_shim_dispatch',
|
||||
controllerRole: 'stream-combined',
|
||||
},
|
||||
})
|
||||
this.cleanupCombinedSignal = combined.cleanup
|
||||
this.generator = this.makeGenerator(combined.signal)
|
||||
@@ -138,12 +158,20 @@ export class OpenAIShimStream implements AsyncIterable<AnthropicStreamEvent> {
|
||||
async *[Symbol.asyncIterator](): AsyncGenerator<AnthropicStreamEvent> {
|
||||
const generator = this.getGenerator()
|
||||
let completed = false
|
||||
let failed = false
|
||||
try {
|
||||
yield* generator
|
||||
completed = true
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
if (!completed && !this.controller.signal.aborted) {
|
||||
this.controller.abort()
|
||||
if (!completed && !failed && !this.controller.signal.aborted) {
|
||||
requestAbort(this.controller, undefined, {
|
||||
source: 'iterator_closed',
|
||||
subsystem: 'openai_shim_dispatch',
|
||||
controllerRole: 'stream-controller',
|
||||
})
|
||||
}
|
||||
this.cleanupCombinedSignal?.()
|
||||
this.cleanupCombinedSignal = undefined
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { expect, test } from 'bun:test'
|
||||
import {
|
||||
createProviderStreamTrace,
|
||||
createReaderCanceller,
|
||||
createStreamAbortError,
|
||||
readWithIdleTimeout,
|
||||
@@ -8,6 +9,7 @@ import {
|
||||
import { geminiSseToAnthropic } from './geminiStreamConversion.js'
|
||||
|
||||
const dependencies = {
|
||||
createProviderStreamTrace,
|
||||
createReaderCanceller,
|
||||
createStreamAbortError,
|
||||
getStreamIdleTimeoutMs: () => 1_000,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { buildAnthropicUsageFromRawUsage } from '../cacheMetrics.js'
|
||||
import type { AnthropicStreamEvent, AnthropicUsage } from '../codexShim.js'
|
||||
import { logForDebugging } from '../../../utils/debug.js'
|
||||
import type { createProviderStreamTrace } from './streamControl.js'
|
||||
|
||||
type ReaderCanceller = {
|
||||
cancel(error?: unknown): void
|
||||
@@ -16,6 +17,7 @@ export type GeminiStreamDependencies = {
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
signal?: AbortSignal,
|
||||
): ReaderCanceller
|
||||
createProviderStreamTrace: typeof createProviderStreamTrace
|
||||
createStreamAbortError(): DOMException
|
||||
getStreamIdleTimeoutMs(): number
|
||||
makeMessageId(): string
|
||||
@@ -25,7 +27,7 @@ export type GeminiStreamDependencies = {
|
||||
options?: {
|
||||
signal?: AbortSignal
|
||||
cancelReader?: (error?: unknown) => void
|
||||
onTimeout?: () => void
|
||||
onTimeout?: (error: unknown) => void
|
||||
},
|
||||
): Promise<StreamReadResult>
|
||||
throwIfStreamAborted(signal?: AbortSignal): void
|
||||
@@ -39,6 +41,7 @@ export async function* geminiSseToAnthropic(
|
||||
): AsyncGenerator<AnthropicStreamEvent> {
|
||||
const {
|
||||
createReaderCanceller,
|
||||
createProviderStreamTrace,
|
||||
createStreamAbortError,
|
||||
getStreamIdleTimeoutMs,
|
||||
makeMessageId,
|
||||
@@ -59,8 +62,11 @@ export async function* geminiSseToAnthropic(
|
||||
let usage: Partial<AnthropicUsage> | undefined
|
||||
let finishReason: string | undefined
|
||||
const streamIdleTimeoutMs = getStreamIdleTimeoutMs()
|
||||
let lastDataTime = Date.now()
|
||||
let lastDataTime = performance.now()
|
||||
let streamComplete = false
|
||||
let protocolComplete = false
|
||||
let readerFailed = false
|
||||
const streamTrace = createProviderStreamTrace('gemini_sse')
|
||||
|
||||
const emitMessageStart = async function* () {
|
||||
if (hasEmittedStart) return
|
||||
@@ -98,20 +104,23 @@ export async function* geminiSseToAnthropic(
|
||||
{
|
||||
signal,
|
||||
cancelReader: readerCanceller.cancel,
|
||||
onTimeout: () => {
|
||||
const elapsed = Math.round((Date.now() - lastDataTime) / 1000)
|
||||
onTimeout: error => {
|
||||
const elapsed = Math.round((performance.now() - lastDataTime) / 1000)
|
||||
logForDebugging(
|
||||
`Gemini SSE stream idle for ${elapsed}s (limit: ${streamIdleTimeoutMs / 1000}s). Connection likely dropped.`,
|
||||
{ level: 'error' },
|
||||
)
|
||||
streamTrace.recordIdleTimeout(error)
|
||||
},
|
||||
},
|
||||
)
|
||||
if (done) {
|
||||
streamComplete = true
|
||||
streamTrace.recordEof()
|
||||
break
|
||||
}
|
||||
if (value) lastDataTime = Date.now()
|
||||
if (value) lastDataTime = performance.now()
|
||||
streamTrace.recordRaw(value)
|
||||
|
||||
throwIfStreamAborted(signal)
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
@@ -122,10 +131,15 @@ export async function* geminiSseToAnthropic(
|
||||
throwIfStreamAborted(signal)
|
||||
const lines = chunk.split('\n').map(line => line.trim()).filter(Boolean)
|
||||
const dataLines = lines.filter(line => line.startsWith('data: '))
|
||||
if (dataLines.length === 0) continue
|
||||
if (dataLines.length === 0) {
|
||||
streamTrace.recordControl()
|
||||
continue
|
||||
}
|
||||
|
||||
const rawData = dataLines.map(line => line.slice(6)).join('\n')
|
||||
if (rawData === '[DONE]') {
|
||||
streamTrace.recordControl()
|
||||
protocolComplete = true
|
||||
yield* emitMessageStart()
|
||||
if (hasEmittedTextStart || hasEmittedCurrentTool) {
|
||||
throwIfStreamAborted(signal)
|
||||
@@ -149,8 +163,15 @@ export async function* geminiSseToAnthropic(
|
||||
|
||||
let parsed: Record<string, unknown>
|
||||
try {
|
||||
parsed = JSON.parse(rawData) as Record<string, unknown>
|
||||
const value: unknown = JSON.parse(rawData)
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
streamTrace.recordIgnored()
|
||||
} else {
|
||||
streamTrace.recordParsed()
|
||||
}
|
||||
parsed = value as Record<string, unknown>
|
||||
} catch {
|
||||
streamTrace.recordIgnored()
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -173,6 +194,7 @@ export async function* geminiSseToAnthropic(
|
||||
const candidate = candidates[0]
|
||||
if (typeof candidate.finishReason === 'string') {
|
||||
finishReason = candidate.finishReason
|
||||
protocolComplete = true
|
||||
}
|
||||
|
||||
const content = candidate.content as
|
||||
@@ -267,8 +289,22 @@ export async function* geminiSseToAnthropic(
|
||||
}
|
||||
throwIfStreamAborted(signal)
|
||||
yield { type: 'message_stop' }
|
||||
streamComplete = true
|
||||
} catch (error) {
|
||||
readerFailed = true
|
||||
throw error
|
||||
} finally {
|
||||
streamTrace.recordClosed(
|
||||
signal?.aborted
|
||||
? 'root_aborted'
|
||||
: readerFailed
|
||||
? 'failed'
|
||||
: protocolComplete
|
||||
? 'complete'
|
||||
: streamComplete
|
||||
? 'eof_without_terminal'
|
||||
: 'incomplete',
|
||||
signal,
|
||||
)
|
||||
if (!streamComplete || signal?.aborted) {
|
||||
readerCanceller.cancel(createStreamAbortError())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
import { afterAll, afterEach, beforeAll, expect, test } from 'bun:test'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../../../test/sharedMutationLock.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
} from '../../../utils/interruptionTrace.js'
|
||||
import type { AnthropicStreamEvent } from '../codexShim.js'
|
||||
import {
|
||||
anthropicSsePassthrough,
|
||||
geminiSseToAnthropic,
|
||||
openaiStreamToAnthropic,
|
||||
} from './responseAdapters.js'
|
||||
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
const originalIdleTimeout = process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS
|
||||
|
||||
beforeAll(async () => {
|
||||
await acquireSharedMutationLock('providerStreamInterruptionTrace.test.ts')
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
if (originalIdleTimeout === undefined) {
|
||||
delete process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS
|
||||
} else {
|
||||
process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = originalIdleTimeout
|
||||
}
|
||||
})
|
||||
|
||||
afterAll(() => {
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
if (originalIdleTimeout === undefined) {
|
||||
delete process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS
|
||||
} else {
|
||||
process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = originalIdleTimeout
|
||||
}
|
||||
releaseSharedMutationLock()
|
||||
})
|
||||
|
||||
function responseFromText(text: string): Response {
|
||||
const encoder = new TextEncoder()
|
||||
return new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(text))
|
||||
controller.close()
|
||||
},
|
||||
}),
|
||||
{ headers: { 'content-type': 'text/event-stream' } },
|
||||
)
|
||||
}
|
||||
|
||||
function openResponseFromText(
|
||||
text: string,
|
||||
onCancel?: () => void,
|
||||
): Response {
|
||||
const encoder = new TextEncoder()
|
||||
return new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(text))
|
||||
},
|
||||
cancel() {
|
||||
onCancel?.()
|
||||
},
|
||||
}),
|
||||
{ headers: { 'content-type': 'text/event-stream' } },
|
||||
)
|
||||
}
|
||||
|
||||
async function collect(
|
||||
stream: AsyncIterable<AnthropicStreamEvent>,
|
||||
): Promise<AnthropicStreamEvent[]> {
|
||||
const events: AnthropicStreamEvent[] = []
|
||||
for await (const event of stream) events.push(event)
|
||||
return events
|
||||
}
|
||||
|
||||
test('all non-Codex readers distinguish raw, parsed, control, and ignored frames', async () => {
|
||||
const cases = [
|
||||
{
|
||||
transport: 'openai_chat_completions',
|
||||
text: [
|
||||
': keepalive',
|
||||
'data: not-json',
|
||||
'data: []',
|
||||
'data: {"choices":[{"delta":{"content":"hello"},"finish_reason":"stop"}]}',
|
||||
'data: [DONE]',
|
||||
'',
|
||||
].join('\n'),
|
||||
expected: { parsed: 1, control: 2, ignored: 2 },
|
||||
stream: (text: string) =>
|
||||
openaiStreamToAnthropic(responseFromText(text), 'test-model'),
|
||||
},
|
||||
{
|
||||
transport: 'gemini_sse',
|
||||
text: [
|
||||
': keepalive',
|
||||
'',
|
||||
'data: not-json',
|
||||
'',
|
||||
'data: []',
|
||||
'',
|
||||
'data: {"candidates":[{"content":{"parts":[{"text":"hello"}]},"finishReason":"STOP"}]}',
|
||||
'',
|
||||
'data: [DONE]',
|
||||
'',
|
||||
'',
|
||||
].join('\n'),
|
||||
expected: { parsed: 1, control: 2, ignored: 2 },
|
||||
stream: (text: string) =>
|
||||
geminiSseToAnthropic(responseFromText(text), 'gemini-test'),
|
||||
},
|
||||
{
|
||||
transport: 'anthropic_messages',
|
||||
text: [
|
||||
': keepalive',
|
||||
'',
|
||||
'data: not-json',
|
||||
'',
|
||||
'data: {}',
|
||||
'',
|
||||
'data: {"type":"message_stop"}',
|
||||
'',
|
||||
'data: [DONE]',
|
||||
'',
|
||||
'',
|
||||
].join('\n'),
|
||||
expected: { parsed: 1, control: 2, ignored: 2 },
|
||||
stream: (text: string) =>
|
||||
anthropicSsePassthrough(responseFromText(text), 'claude-test'),
|
||||
},
|
||||
]
|
||||
|
||||
for (const scenario of cases) {
|
||||
__resetInterruptionTraceForTests()
|
||||
await collect(scenario.stream(scenario.text))
|
||||
|
||||
const closed = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'provider_stream.reader_closed' &&
|
||||
entry.transport === scenario.transport,
|
||||
)
|
||||
expect(closed).toMatchObject({
|
||||
outcome: 'complete',
|
||||
rawByteCount: new TextEncoder().encode(scenario.text).byteLength,
|
||||
parsedFrameCount: scenario.expected.parsed,
|
||||
controlFrameCount: scenario.expected.control,
|
||||
ignoredFrameCount: scenario.expected.ignored,
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
for (const traceEnabled of [false, true]) {
|
||||
test(`OpenAI-compatible and Gemini null payloads fail with tracing ${traceEnabled ? 'enabled' : 'disabled'}`, async () => {
|
||||
const previousTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
try {
|
||||
if (traceEnabled) process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
else delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
|
||||
const cases = [
|
||||
{
|
||||
transport: 'openai_chat_completions',
|
||||
stream: () =>
|
||||
openaiStreamToAnthropic(
|
||||
responseFromText('data: null\n\ndata: [DONE]\n\n'),
|
||||
'test-model',
|
||||
),
|
||||
},
|
||||
{
|
||||
transport: 'gemini_sse',
|
||||
stream: () =>
|
||||
geminiSseToAnthropic(
|
||||
responseFromText('data: null\n\ndata: [DONE]\n\n'),
|
||||
'gemini-test',
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
for (const scenario of cases) {
|
||||
__resetInterruptionTraceForTests()
|
||||
await expect(collect(scenario.stream())).rejects.toBeInstanceOf(TypeError)
|
||||
const snapshot = __getInterruptionTraceSnapshotForTests()
|
||||
if (!traceEnabled) {
|
||||
expect(snapshot).toEqual([])
|
||||
continue
|
||||
}
|
||||
expect(
|
||||
snapshot.find(
|
||||
entry =>
|
||||
entry.event === 'provider_stream.reader_closed' &&
|
||||
entry.transport === scenario.transport,
|
||||
),
|
||||
).toMatchObject({
|
||||
outcome: 'failed',
|
||||
ignoredFrameCount: 1,
|
||||
})
|
||||
}
|
||||
} finally {
|
||||
__resetInterruptionTraceForTests()
|
||||
if (previousTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = previousTrace
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
test('Anthropic terminal consumer closure cancels an open response body', async () => {
|
||||
__resetInterruptionTraceForTests()
|
||||
let cancellations = 0
|
||||
const iterator = anthropicSsePassthrough(
|
||||
openResponseFromText(
|
||||
'data: {"type":"message_stop"}\n\n',
|
||||
() => {
|
||||
cancellations++
|
||||
},
|
||||
),
|
||||
'claude-test',
|
||||
)[Symbol.asyncIterator]()
|
||||
|
||||
expect((await iterator.next()).value?.type).toBe('message_stop')
|
||||
await iterator.return?.(undefined)
|
||||
expect(cancellations).toBe(1)
|
||||
})
|
||||
|
||||
test('all non-Codex readers report a failure after terminal evidence then timeout', async () => {
|
||||
process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = '20'
|
||||
const cases = [
|
||||
{
|
||||
transport: 'openai_chat_completions',
|
||||
stream: () =>
|
||||
openaiStreamToAnthropic(
|
||||
openResponseFromText(
|
||||
'data: {"choices":[{"delta":{"content":"hello"},"finish_reason":"stop"}]}\n',
|
||||
),
|
||||
'test-model',
|
||||
),
|
||||
},
|
||||
{
|
||||
transport: 'gemini_sse',
|
||||
stream: () =>
|
||||
geminiSseToAnthropic(
|
||||
openResponseFromText(
|
||||
'data: {"candidates":[{"content":{"parts":[{"text":"hello"}]},"finishReason":"STOP"}]}\n\n',
|
||||
),
|
||||
'gemini-test',
|
||||
),
|
||||
},
|
||||
{
|
||||
transport: 'anthropic_messages',
|
||||
stream: () =>
|
||||
anthropicSsePassthrough(
|
||||
openResponseFromText('data: {"type":"message_stop"}\n\n'),
|
||||
'claude-test',
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
for (const scenario of cases) {
|
||||
__resetInterruptionTraceForTests()
|
||||
await expect(collect(scenario.stream())).rejects.toBeDefined()
|
||||
const closed = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'provider_stream.reader_closed' &&
|
||||
entry.transport === scenario.transport,
|
||||
)
|
||||
expect(closed?.outcome).toBe('failed')
|
||||
}
|
||||
})
|
||||
|
||||
test('all non-Codex readers distinguish transport EOF from protocol completion', async () => {
|
||||
const cases = [
|
||||
{
|
||||
transport: 'openai_chat_completions',
|
||||
stream: () =>
|
||||
openaiStreamToAnthropic(
|
||||
responseFromText(
|
||||
'data: {"choices":[{"delta":{"content":"hello"},"finish_reason":null}]}\n',
|
||||
),
|
||||
'test-model',
|
||||
),
|
||||
},
|
||||
{
|
||||
transport: 'gemini_sse',
|
||||
stream: () =>
|
||||
geminiSseToAnthropic(
|
||||
responseFromText(
|
||||
'data: {"candidates":[{"content":{"parts":[{"text":"hello"}]}}]}\n\n',
|
||||
),
|
||||
'gemini-test',
|
||||
),
|
||||
},
|
||||
{
|
||||
transport: 'anthropic_messages',
|
||||
stream: () =>
|
||||
anthropicSsePassthrough(
|
||||
responseFromText(
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}\n\n',
|
||||
),
|
||||
'claude-test',
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
for (const scenario of cases) {
|
||||
__resetInterruptionTraceForTests()
|
||||
await collect(scenario.stream())
|
||||
|
||||
const closed = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'provider_stream.reader_closed' &&
|
||||
entry.transport === scenario.transport,
|
||||
)
|
||||
expect(closed?.outcome).toBe('eof_without_terminal')
|
||||
}
|
||||
})
|
||||
|
||||
test('all non-Codex reader idle errors carry the provider timeout causal event', async () => {
|
||||
process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = '20'
|
||||
const cases = [
|
||||
{
|
||||
transport: 'openai_chat_completions',
|
||||
stream: (response: Response) =>
|
||||
openaiStreamToAnthropic(response, 'test-model'),
|
||||
},
|
||||
{
|
||||
transport: 'gemini_sse',
|
||||
stream: (response: Response) =>
|
||||
geminiSseToAnthropic(response, 'gemini-test'),
|
||||
},
|
||||
{
|
||||
transport: 'anthropic_messages',
|
||||
stream: (response: Response) =>
|
||||
anthropicSsePassthrough(response, 'claude-test'),
|
||||
},
|
||||
]
|
||||
|
||||
for (const scenario of cases) {
|
||||
__resetInterruptionTraceForTests()
|
||||
const response = new Response(new ReadableStream<Uint8Array>())
|
||||
let caught: unknown
|
||||
try {
|
||||
await collect(scenario.stream(response))
|
||||
} catch (error) {
|
||||
caught = error
|
||||
}
|
||||
|
||||
const idleTimeout = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'provider_stream.idle_timeout' &&
|
||||
entry.transport === scenario.transport,
|
||||
)
|
||||
const readerClosed = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'provider_stream.reader_closed' &&
|
||||
entry.transport === scenario.transport,
|
||||
)
|
||||
expect(caught).toBeDefined()
|
||||
expect(idleTimeout).toBeDefined()
|
||||
expect(readerClosed?.causalEventId).toBe(idleTimeout?.eventId)
|
||||
}
|
||||
})
|
||||
@@ -25,6 +25,7 @@ import { openaiStreamToAnthropic as convertOpenAIStream } from './streamConversi
|
||||
import { geminiSseToAnthropic as convertGeminiStream } from './geminiStreamConversion.js'
|
||||
import {
|
||||
anthropicSsePassthrough as parseAnthropicSsePassthrough,
|
||||
createProviderStreamTrace,
|
||||
createReaderCanceller,
|
||||
createStreamAbortError,
|
||||
getStreamIdleTimeoutMs,
|
||||
@@ -93,6 +94,7 @@ export async function* geminiSseToAnthropic(
|
||||
signal?: AbortSignal,
|
||||
): AsyncGenerator<AnthropicStreamEvent> {
|
||||
yield* convertGeminiStream(response, model, signal, {
|
||||
createProviderStreamTrace,
|
||||
createReaderCanceller,
|
||||
createStreamAbortError,
|
||||
getStreamIdleTimeoutMs,
|
||||
@@ -135,6 +137,7 @@ export async function* openaiStreamToAnthropic(
|
||||
streamModel,
|
||||
),
|
||||
couldBeRawToolCallsRequestedPrefix,
|
||||
createProviderStreamTrace,
|
||||
createReaderCanceller,
|
||||
createStreamAbortError,
|
||||
findXmlToolCallOpener,
|
||||
|
||||
@@ -142,6 +142,28 @@ test('readWithIdleTimeout settles when a custom canceller throws synchronously',
|
||||
reader.releaseLock()
|
||||
})
|
||||
|
||||
test('readWithIdleTimeout contains a throwing custom error factory', async () => {
|
||||
const cancelReasons: unknown[] = []
|
||||
const reader = new ReadableStream<Uint8Array>({
|
||||
cancel(reason) {
|
||||
cancelReasons.push(reason)
|
||||
},
|
||||
}).getReader()
|
||||
|
||||
await expect(
|
||||
withDeadline(
|
||||
readWithIdleTimeout(reader, 20, {
|
||||
createTimeoutError: () => {
|
||||
throw new Error('factory failed')
|
||||
},
|
||||
}),
|
||||
'idle timeout did not reject within 500ms',
|
||||
),
|
||||
).rejects.toBeInstanceOf(StreamIdleTimeoutError)
|
||||
expect(cancelReasons).toHaveLength(1)
|
||||
expect(cancelReasons[0]).toBeInstanceOf(StreamIdleTimeoutError)
|
||||
})
|
||||
|
||||
test('stream idle timeout parser validates and bounds overrides', () => {
|
||||
expect(getStreamIdleTimeoutMs()).toBe(90_000)
|
||||
process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS = '25'
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
import {
|
||||
flushInterruptionTrace,
|
||||
getInterruptionSignalAbortEventId,
|
||||
setInterruptionErrorCausalEventId,
|
||||
traceInterruptionEvent,
|
||||
} from '../../../utils/interruptionTrace.js'
|
||||
|
||||
const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 90_000
|
||||
const MAX_STREAM_IDLE_TIMEOUT_MS = 2_147_483_647
|
||||
|
||||
@@ -57,13 +64,79 @@ export function getStreamIdleTimeoutMs(): number {
|
||||
: DEFAULT_STREAM_IDLE_TIMEOUT_MS
|
||||
}
|
||||
|
||||
export function createProviderStreamTrace(transport: string) {
|
||||
let lastRawByteTime = performance.now()
|
||||
let lastParsedFrameTime = lastRawByteTime
|
||||
let rawByteCount = 0
|
||||
let parsedFrameCount = 0
|
||||
let controlFrameCount = 0
|
||||
let ignoredFrameCount = 0
|
||||
let causalEventId: string | undefined
|
||||
|
||||
const fields = () => ({
|
||||
subsystem: 'provider_stream',
|
||||
transport,
|
||||
rawByteCount,
|
||||
parsedFrameCount,
|
||||
controlFrameCount,
|
||||
ignoredFrameCount,
|
||||
})
|
||||
|
||||
traceInterruptionEvent('provider_stream.read_started', fields())
|
||||
|
||||
return {
|
||||
recordRaw(value: Uint8Array | undefined): void {
|
||||
if (!value || value.byteLength === 0) return
|
||||
lastRawByteTime = performance.now()
|
||||
rawByteCount += value.byteLength
|
||||
if (rawByteCount === value.byteLength) {
|
||||
traceInterruptionEvent('provider_stream.first_raw_byte', fields())
|
||||
}
|
||||
},
|
||||
recordParsed(): void {
|
||||
lastParsedFrameTime = performance.now()
|
||||
parsedFrameCount++
|
||||
},
|
||||
recordControl(): void {
|
||||
controlFrameCount++
|
||||
},
|
||||
recordIgnored(): void {
|
||||
ignoredFrameCount++
|
||||
},
|
||||
recordIdleTimeout(error: unknown): void {
|
||||
const now = performance.now()
|
||||
causalEventId = traceInterruptionEvent('provider_stream.idle_timeout', {
|
||||
...fields(),
|
||||
sinceLastRawByteMs: now - lastRawByteTime,
|
||||
sinceLastParsedFrameMs: now - lastParsedFrameTime,
|
||||
})
|
||||
setInterruptionErrorCausalEventId(error, causalEventId)
|
||||
flushInterruptionTrace('provider_stream_idle_timeout')
|
||||
},
|
||||
recordEof(): void {
|
||||
traceInterruptionEvent('provider_stream.eof', fields())
|
||||
},
|
||||
recordClosed(outcome: string, signal?: AbortSignal): void {
|
||||
traceInterruptionEvent('provider_stream.reader_closed', {
|
||||
...fields(),
|
||||
outcome,
|
||||
reason: signal?.reason,
|
||||
causalEventId:
|
||||
(signal && getInterruptionSignalAbortEventId(signal)) ??
|
||||
causalEventId,
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export async function readWithIdleTimeout(
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
timeoutMs: number,
|
||||
options: {
|
||||
signal?: AbortSignal
|
||||
cancelReader?: (error?: unknown) => void
|
||||
onTimeout?: () => void
|
||||
onTimeout?: (error: unknown) => void
|
||||
createTimeoutError?: () => unknown
|
||||
} = {},
|
||||
): Promise<StreamReadResult> {
|
||||
const signal = options.signal
|
||||
@@ -111,9 +184,15 @@ export async function readWithIdleTimeout(
|
||||
}
|
||||
|
||||
timeoutId = setTimeout(() => {
|
||||
const error = new StreamIdleTimeoutError(timeoutMs)
|
||||
let error: unknown
|
||||
try {
|
||||
options.onTimeout?.()
|
||||
error = options.createTimeoutError?.()
|
||||
} catch {
|
||||
// Fall back to the standard timeout error.
|
||||
}
|
||||
error ??= new StreamIdleTimeoutError(timeoutMs)
|
||||
try {
|
||||
options.onTimeout?.(error)
|
||||
} catch {
|
||||
// Ignore diagnostic callback failures.
|
||||
}
|
||||
@@ -140,27 +219,33 @@ export async function* anthropicSsePassthrough<T extends object>(
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ''
|
||||
const streamIdleTimeoutMs = getStreamIdleTimeoutMs()
|
||||
let lastDataTime = Date.now()
|
||||
let lastDataTime = performance.now()
|
||||
let streamComplete = false
|
||||
let protocolComplete = false
|
||||
let readerFailed = false
|
||||
const streamTrace = createProviderStreamTrace('anthropic_messages')
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await readWithIdleTimeout(reader, streamIdleTimeoutMs, {
|
||||
signal,
|
||||
cancelReader: readerCanceller.cancel,
|
||||
onTimeout: () => {
|
||||
const elapsed = Math.round((Date.now() - lastDataTime) / 1000)
|
||||
onTimeout: error => {
|
||||
const elapsed = Math.round((performance.now() - lastDataTime) / 1000)
|
||||
logForDebugging(
|
||||
`Anthropic-compatible SSE stream idle for ${elapsed}s (limit: ${streamIdleTimeoutMs / 1000}s). Connection likely dropped.`,
|
||||
{ level: 'error' },
|
||||
)
|
||||
streamTrace.recordIdleTimeout(error)
|
||||
},
|
||||
})
|
||||
if (done) {
|
||||
streamComplete = true
|
||||
streamTrace.recordEof()
|
||||
buffer += decoder.decode()
|
||||
} else {
|
||||
if (value) lastDataTime = Date.now()
|
||||
if (value) lastDataTime = performance.now()
|
||||
streamTrace.recordRaw(value)
|
||||
throwIfStreamAborted(signal)
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
}
|
||||
@@ -170,10 +255,14 @@ export async function* anthropicSsePassthrough<T extends object>(
|
||||
throwIfStreamAborted(signal)
|
||||
const lines = chunk.split(/\r\n|\n|\r/).map(line => line.trim()).filter(Boolean)
|
||||
const dataLines = lines.filter(line => line.startsWith('data:'))
|
||||
if (dataLines.length === 0) continue
|
||||
if (dataLines.length === 0) {
|
||||
streamTrace.recordControl()
|
||||
continue
|
||||
}
|
||||
const rawData = dataLines.map(line => line.slice(5).replace(/^ /, '')).join('\n')
|
||||
if (rawData === '[DONE]') {
|
||||
streamComplete = true
|
||||
streamTrace.recordControl()
|
||||
protocolComplete = true
|
||||
readerCanceller.cancel()
|
||||
return
|
||||
}
|
||||
@@ -182,17 +271,41 @@ export async function* anthropicSsePassthrough<T extends object>(
|
||||
parsed = JSON.parse(rawData) as T
|
||||
} catch {
|
||||
// Ignore malformed frames and continue parsing later frames.
|
||||
streamTrace.recordIgnored()
|
||||
continue
|
||||
}
|
||||
if (parsed && typeof parsed === 'object' && 'type' in parsed) {
|
||||
streamTrace.recordParsed()
|
||||
if ((parsed as { type?: unknown }).type === 'message_stop') {
|
||||
protocolComplete = true
|
||||
}
|
||||
throwIfStreamAborted(signal)
|
||||
yield parsed as Awaited<T>
|
||||
} else {
|
||||
streamTrace.recordIgnored()
|
||||
}
|
||||
}
|
||||
if (done) break
|
||||
}
|
||||
} catch (error) {
|
||||
readerFailed = true
|
||||
throw error
|
||||
} finally {
|
||||
if (!streamComplete || signal?.aborted) readerCanceller.cancel(createStreamAbortError())
|
||||
streamTrace.recordClosed(
|
||||
signal?.aborted
|
||||
? 'root_aborted'
|
||||
: readerFailed
|
||||
? 'failed'
|
||||
: protocolComplete
|
||||
? 'complete'
|
||||
: streamComplete
|
||||
? 'eof_without_terminal'
|
||||
: 'incomplete',
|
||||
signal,
|
||||
)
|
||||
if (!streamComplete || signal?.aborted) {
|
||||
readerCanceller.cancel(createStreamAbortError())
|
||||
}
|
||||
readerCanceller.cleanup()
|
||||
reader.releaseLock()
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
geminiSseToAnthropic,
|
||||
type GeminiStreamDependencies,
|
||||
} from './geminiStreamConversion.js'
|
||||
import { createProviderStreamTrace } from './streamControl.js'
|
||||
|
||||
function makeSseResponse(frames: unknown[]): Response {
|
||||
const encoder = new TextEncoder()
|
||||
@@ -77,6 +78,7 @@ async function readWithSignal(
|
||||
}
|
||||
|
||||
const commonControlDependencies: GeminiStreamDependencies = {
|
||||
createProviderStreamTrace,
|
||||
createReaderCanceller,
|
||||
createStreamAbortError: () => new DOMException('Aborted', 'AbortError'),
|
||||
getStreamIdleTimeoutMs: () => 1_000,
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
getStreamStats,
|
||||
processStreamChunk,
|
||||
} from '../../../utils/streamingOptimizer.js'
|
||||
import type { createProviderStreamTrace } from './streamControl.js'
|
||||
|
||||
type ParsedRawToolCall = { id: string; name: string; argumentsJson: string }
|
||||
type ParsedTextToolCall = { id: string; name: string; arguments: unknown }
|
||||
@@ -65,6 +66,7 @@ export type StreamConversionDependencies = {
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
signal?: AbortSignal,
|
||||
): ReaderCanceller
|
||||
createProviderStreamTrace: typeof createProviderStreamTrace
|
||||
createStreamAbortError(): DOMException
|
||||
findXmlToolCallOpener(text: string, allowHy3: boolean): number
|
||||
geminiThoughtSignatureFromExtraContent(extraContent: unknown): string | undefined
|
||||
@@ -85,7 +87,7 @@ export type StreamConversionDependencies = {
|
||||
options?: {
|
||||
signal?: AbortSignal
|
||||
cancelReader?: (error?: unknown) => void
|
||||
onTimeout?: () => void
|
||||
onTimeout?: (error: unknown) => void
|
||||
},
|
||||
): Promise<StreamReadResult>
|
||||
repairPossiblyTruncatedObjectJson(raw: string): string | null
|
||||
@@ -116,6 +118,7 @@ export async function* openaiStreamToAnthropic(
|
||||
convertNonStreamingResponseToAnthropicMessage,
|
||||
couldBeRawToolCallsRequestedPrefix,
|
||||
createReaderCanceller,
|
||||
createProviderStreamTrace,
|
||||
createStreamAbortError,
|
||||
findXmlToolCallOpener,
|
||||
geminiThoughtSignatureFromExtraContent,
|
||||
@@ -154,6 +157,8 @@ export async function* openaiStreamToAnthropic(
|
||||
let lastStopReason: 'tool_use' | 'max_tokens' | 'end_turn' | null = null
|
||||
let hasEmittedFinalUsage = false
|
||||
let protocolComplete = false
|
||||
let providerTerminalObserved = false
|
||||
let readerFailed = false
|
||||
let hasProcessedFinishReason = false
|
||||
// Accumulated text for Ollama text-based tool call fallback parsing (#1053)
|
||||
let accumulatedText = ''
|
||||
@@ -299,8 +304,9 @@ export async function* openaiStreamToAnthropic(
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ''
|
||||
const streamIdleTimeoutMs = getStreamIdleTimeoutMs()
|
||||
let lastDataTime = Date.now()
|
||||
let lastDataTime = performance.now()
|
||||
let streamComplete = false
|
||||
const streamTrace = createProviderStreamTrace('openai_chat_completions')
|
||||
|
||||
const closeActiveContentBlock = async function* () {
|
||||
if (!hasEmittedContentStart) return
|
||||
@@ -463,19 +469,22 @@ export async function* openaiStreamToAnthropic(
|
||||
const { done, value } = await readWithIdleTimeout(reader, streamIdleTimeoutMs, {
|
||||
signal,
|
||||
cancelReader: readerCanceller.cancel,
|
||||
onTimeout: () => {
|
||||
const elapsed = Math.round((Date.now() - lastDataTime) / 1000)
|
||||
onTimeout: error => {
|
||||
const elapsed = Math.round((performance.now() - lastDataTime) / 1000)
|
||||
logForDebugging(
|
||||
`OpenAI-compatible SSE stream idle for ${elapsed}s (limit: ${streamIdleTimeoutMs / 1000}s). Connection likely dropped.`,
|
||||
{ level: 'error' },
|
||||
)
|
||||
streamTrace.recordIdleTimeout(error)
|
||||
},
|
||||
})
|
||||
if (done) {
|
||||
streamComplete = true
|
||||
streamTrace.recordEof()
|
||||
break
|
||||
}
|
||||
if (value) lastDataTime = Date.now()
|
||||
if (value) lastDataTime = performance.now()
|
||||
streamTrace.recordRaw(value)
|
||||
|
||||
throwIfStreamAborted(signal)
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
@@ -487,17 +496,32 @@ export async function* openaiStreamToAnthropic(
|
||||
const trimmed = line.trim()
|
||||
if (!trimmed) continue
|
||||
if (trimmed === 'data: [DONE]') {
|
||||
streamTrace.recordControl()
|
||||
// `[DONE]` is protocol completion; do not wait for a transport EOF.
|
||||
// Leave streamComplete false so finally cancels an unread response body.
|
||||
protocolComplete = true
|
||||
break
|
||||
}
|
||||
if (!trimmed.startsWith('data: ')) continue
|
||||
if (!trimmed.startsWith('data: ')) {
|
||||
if (trimmed.startsWith(':') || trimmed.startsWith('event:')) {
|
||||
streamTrace.recordControl()
|
||||
} else {
|
||||
streamTrace.recordIgnored()
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
let chunk: OpenAIStreamChunk
|
||||
try {
|
||||
chunk = JSON.parse(trimmed.slice(6))
|
||||
const parsed: unknown = JSON.parse(trimmed.slice(6))
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
streamTrace.recordIgnored()
|
||||
} else {
|
||||
streamTrace.recordParsed()
|
||||
}
|
||||
chunk = parsed as OpenAIStreamChunk
|
||||
} catch {
|
||||
streamTrace.recordIgnored()
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -752,6 +776,7 @@ export async function* openaiStreamToAnthropic(
|
||||
// Finish — guard ensures we only process finish_reason once even if
|
||||
// multiple chunks arrive with finish_reason set (some providers do this)
|
||||
if (choice.finish_reason && !hasProcessedFinishReason) {
|
||||
providerTerminalObserved = true
|
||||
hasProcessedFinishReason = true
|
||||
|
||||
// Close any open thinking block that wasn't closed by content transition
|
||||
@@ -1083,7 +1108,22 @@ export async function* openaiStreamToAnthropic(
|
||||
if (protocolComplete) break
|
||||
}
|
||||
yield* finalizeIncompleteStream()
|
||||
} catch (error) {
|
||||
readerFailed = true
|
||||
throw error
|
||||
} finally {
|
||||
streamTrace.recordClosed(
|
||||
signal?.aborted
|
||||
? 'root_aborted'
|
||||
: readerFailed
|
||||
? 'failed'
|
||||
: protocolComplete || providerTerminalObserved
|
||||
? 'complete'
|
||||
: streamComplete
|
||||
? 'eof_without_terminal'
|
||||
: 'incomplete',
|
||||
signal,
|
||||
)
|
||||
if (!streamComplete || signal?.aborted) {
|
||||
readerCanceller.cancel(createStreamAbortError())
|
||||
}
|
||||
|
||||
@@ -9,6 +9,16 @@ import {
|
||||
redactSecretSubstringsForDisplay,
|
||||
} from '../../../utils/providerSecrets.js'
|
||||
import { redactUrlForDisplay } from '../../../utils/redaction.js'
|
||||
import {
|
||||
getInterruptionSignalAbortEventId,
|
||||
getInterruptionSignalId,
|
||||
registerInterruptionController,
|
||||
registerInterruptionSignal,
|
||||
requestAbort,
|
||||
traceCombinedAbortSignal,
|
||||
traceCombinedSignal,
|
||||
traceInterruptionEvent,
|
||||
} from '../../../utils/interruptionTrace.js'
|
||||
import {
|
||||
buildOpenAICompatibilityErrorMessage,
|
||||
classifyOpenAINetworkFailure,
|
||||
@@ -82,28 +92,69 @@ function combineRequestSignals(
|
||||
}
|
||||
|
||||
if (typeof AbortSignal.any === 'function') {
|
||||
const signal = AbortSignal.any([callerSignal, deadlineSignal])
|
||||
traceCombinedAbortSignal(signal, [callerSignal, deadlineSignal], {
|
||||
subsystem: 'openai_shim_transport',
|
||||
controllerRole: 'request-combined',
|
||||
})
|
||||
return {
|
||||
signal: AbortSignal.any([callerSignal, deadlineSignal]),
|
||||
signal,
|
||||
cleanupAfterHeaders: () => {},
|
||||
cleanup: () => {},
|
||||
}
|
||||
}
|
||||
|
||||
const combined = new AbortController()
|
||||
const callerId = registerInterruptionSignal(callerSignal, {
|
||||
subsystem: 'openai_shim_transport',
|
||||
controllerRole: 'request-caller',
|
||||
})
|
||||
const deadlineId = registerInterruptionSignal(deadlineSignal, {
|
||||
subsystem: 'openai_shim_transport',
|
||||
controllerRole: 'headers-deadline',
|
||||
})
|
||||
const parentControllerIds = [callerId, deadlineId].filter(
|
||||
(id): id is string => id !== undefined,
|
||||
)
|
||||
traceCombinedSignal(combined, [callerSignal, deadlineSignal], {
|
||||
subsystem: 'openai_shim_transport',
|
||||
controllerRole: 'request-combined',
|
||||
})
|
||||
const abortFromCaller = () => {
|
||||
deadlineSignal.removeEventListener('abort', abortFromDeadline)
|
||||
combined.abort(callerSignal.reason)
|
||||
requestAbort(combined, callerSignal.reason, {
|
||||
source: 'request_caller',
|
||||
subsystem: 'openai_shim_transport',
|
||||
controllerRole: 'request-combined',
|
||||
parentControllerIds,
|
||||
winningParentControllerId: getInterruptionSignalId(callerSignal),
|
||||
causalEventId: getInterruptionSignalAbortEventId(callerSignal),
|
||||
})
|
||||
}
|
||||
const abortFromDeadline = () => {
|
||||
callerSignal.removeEventListener('abort', abortFromCaller)
|
||||
combined.abort(deadlineSignal.reason)
|
||||
requestAbort(combined, deadlineSignal.reason, {
|
||||
source: 'headers_deadline',
|
||||
subsystem: 'openai_shim_transport',
|
||||
controllerRole: 'request-combined',
|
||||
parentControllerIds,
|
||||
winningParentControllerId: getInterruptionSignalId(deadlineSignal),
|
||||
causalEventId: getInterruptionSignalAbortEventId(deadlineSignal),
|
||||
})
|
||||
}
|
||||
const cleanupAfterHeaders = () => {
|
||||
deadlineSignal.removeEventListener('abort', abortFromDeadline)
|
||||
}
|
||||
let cleanedUp = false
|
||||
const cleanup = () => {
|
||||
if (cleanedUp) return
|
||||
cleanedUp = true
|
||||
callerSignal.removeEventListener('abort', abortFromCaller)
|
||||
cleanupAfterHeaders()
|
||||
traceInterruptionEvent('combined_signal.cleanup', {
|
||||
subsystem: 'openai_shim_transport',
|
||||
controllerRole: 'request-combined',
|
||||
})
|
||||
}
|
||||
|
||||
callerSignal.addEventListener('abort', abortFromCaller, { once: true })
|
||||
@@ -187,6 +238,10 @@ export async function fetchWithHeadersDeadline(
|
||||
const redactedUrl = redactUrlForDiagnostics(url)
|
||||
const fetchWithAttemptDeadline: ProxyRetryFetcher = async (input, attemptInit) => {
|
||||
const deadlineController = new AbortController()
|
||||
registerInterruptionController(deadlineController, {
|
||||
subsystem: 'openai_shim_transport',
|
||||
controllerRole: 'headers-deadline',
|
||||
})
|
||||
const timeoutReason = new ResponseHeadersTimeoutError(
|
||||
options.timeoutMs,
|
||||
redactedUrl,
|
||||
@@ -198,7 +253,12 @@ export async function fetchWithHeadersDeadline(
|
||||
cleanupAfterBody,
|
||||
} = combineRequestSignals(options.callerSignal, deadlineController.signal)
|
||||
const timer = setTimeout(
|
||||
() => deadlineController.abort(timeoutReason),
|
||||
() =>
|
||||
requestAbort(deadlineController, timeoutReason, {
|
||||
source: 'headers_deadline_timer',
|
||||
subsystem: 'openai_shim_transport',
|
||||
controllerRole: 'headers-deadline',
|
||||
}),
|
||||
options.timeoutMs,
|
||||
)
|
||||
timer.unref?.()
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
expect,
|
||||
mock,
|
||||
test,
|
||||
vi,
|
||||
} from 'bun:test'
|
||||
import { randomUUID } from 'crypto'
|
||||
import { unlink } from 'fs/promises'
|
||||
@@ -19,6 +20,11 @@ import {
|
||||
} from '../../test/sharedMutationLock.js'
|
||||
import type { Message } from '../../types/message.js'
|
||||
import * as realConfig from '../../utils/config.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
|
||||
// Several earlier test files in the smoke suite call
|
||||
// mock.module('../../utils/model/providers.js', ...) to stub getAPIProvider.
|
||||
@@ -815,6 +821,51 @@ afterAll(async () => {
|
||||
})
|
||||
|
||||
describe('compactConversation provider gate', () => {
|
||||
test('attributes cache-sharing timeout aborts to the compact fork', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const runForkedAgent = mock(() => new Promise(() => {}))
|
||||
const { compactConversation } = await importCompact({ runForkedAgent })
|
||||
const messages = [userMessage('Hello'), assistantMessage('Hi there!')]
|
||||
const compactPromise = compactConversation(
|
||||
messages,
|
||||
toolUseContext(),
|
||||
cacheSafeParams(messages),
|
||||
false,
|
||||
)
|
||||
|
||||
for (let attempt = 0; attempt < 10 && runForkedAgent.mock.calls.length === 0; attempt++) {
|
||||
await Promise.resolve()
|
||||
}
|
||||
expect(runForkedAgent).toHaveBeenCalledTimes(1)
|
||||
|
||||
vi.advanceTimersByTime(120_000)
|
||||
await compactPromise
|
||||
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'abort.requested',
|
||||
),
|
||||
).toMatchObject({
|
||||
source: 'compact_timeout',
|
||||
subsystem: 'compact',
|
||||
controllerRole: 'compact-fork',
|
||||
})
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('skips forked-agent cache-sharing for non-Anthropic providers', async () => {
|
||||
// Simulate a non-Anthropic provider (e.g. OpenAI) via env vars.
|
||||
// The real isAnthropicProvider() reads from process.env and returns false.
|
||||
|
||||
@@ -61,6 +61,7 @@ import {
|
||||
} from '../../utils/hooks.js'
|
||||
import { logError } from '../../utils/log.js'
|
||||
import { MEMORY_TYPE_VALUES } from '../../utils/memory/types.js'
|
||||
import { requestAbort } from '../../utils/interruptionTrace.js'
|
||||
import {
|
||||
createCompactBoundaryMessage,
|
||||
createUserMessage,
|
||||
@@ -139,6 +140,14 @@ export const POST_COMPACT_SKILLS_TOKEN_BUDGET = 25_000
|
||||
const MAX_COMPACT_STREAMING_RETRIES = 2
|
||||
const COMPACT_TIMEOUT_MS = 120_000
|
||||
|
||||
function requestCompactTimeoutAbort(controller: AbortController): void {
|
||||
requestAbort(controller, undefined, {
|
||||
source: 'compact_timeout',
|
||||
subsystem: 'compact',
|
||||
controllerRole: 'compact-fork',
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip image blocks from user messages before sending for compaction.
|
||||
* Images are not needed for generating a conversation summary and can
|
||||
@@ -1268,7 +1277,10 @@ async function streamCompactSummary({
|
||||
// Use a child AbortController that properly propagates parent aborts
|
||||
// (user ESC) and cleans up listeners automatically via createChildAbortController.
|
||||
const forkAbortController = context.abortController
|
||||
? createChildAbortController(context.abortController)
|
||||
? createChildAbortController(context.abortController, undefined, {
|
||||
subsystem: 'compact',
|
||||
controllerRole: 'compact-fork',
|
||||
})
|
||||
: new AbortController()
|
||||
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined
|
||||
@@ -1298,7 +1310,7 @@ async function streamCompactSummary({
|
||||
}),
|
||||
new Promise<never>((_, reject) => {
|
||||
timeoutId = setTimeout(() => {
|
||||
forkAbortController.abort()
|
||||
requestCompactTimeoutAbort(forkAbortController)
|
||||
reject(new Error('Compaction timed out'))
|
||||
}, COMPACT_TIMEOUT_MS)
|
||||
}),
|
||||
|
||||
@@ -8,6 +8,17 @@ import {
|
||||
} from './controller.js'
|
||||
import type { GoalState } from './types.js'
|
||||
import type { AssistantMessage } from '../../types/message.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
registerInterruptionController,
|
||||
requestAbort,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../../test/sharedMutationLock.js'
|
||||
|
||||
function assistant(uuid: string, text: string) {
|
||||
// Minimal fixture — cast rather than fabricate the full envelope.
|
||||
@@ -57,6 +68,66 @@ async function drain(
|
||||
}
|
||||
|
||||
describe('goal continuation controller', () => {
|
||||
test.serial('traces goal evaluation start and completion', async () => {
|
||||
await acquireSharedMutationLock('goal/controller trace')
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
try {
|
||||
const { context, abortController } = makeContext()
|
||||
registerInterruptionController(abortController, {
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
await drain(
|
||||
evaluateGoalAfterTurn({
|
||||
messagesForQuery: [],
|
||||
assistantMessages: [assistant('assistant-trace', 'Done.')],
|
||||
toolUseContext: context,
|
||||
querySource: 'repl_main_thread',
|
||||
deps: {
|
||||
evaluateGoal: async () => {
|
||||
requestAbort(abortController, 'user-cancel', {
|
||||
source: 'cancel_keybinding',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
return {
|
||||
complete: false,
|
||||
confidence: 0,
|
||||
decision: 'error',
|
||||
reason: 'cancelled',
|
||||
nextInstruction: null,
|
||||
}
|
||||
},
|
||||
saveGoalState: async () => {},
|
||||
},
|
||||
}),
|
||||
)
|
||||
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
expect(trace.map(entry => entry.event)).toEqual(
|
||||
expect.arrayContaining([
|
||||
'goal.evaluation_started',
|
||||
'goal.evaluation_completed',
|
||||
]),
|
||||
)
|
||||
const rootAbort = trace.find(entry => entry.event === 'abort.requested')
|
||||
const completed = trace.find(
|
||||
entry => entry.event === 'goal.evaluation_completed',
|
||||
)
|
||||
expect(rootAbort).toBeDefined()
|
||||
expect(completed).toBeDefined()
|
||||
expect(typeof rootAbort!.eventId).toBe('string')
|
||||
expect(typeof completed!.causalEventId).toBe('string')
|
||||
expect(completed!.causalEventId).toBe(rootAbort!.eventId)
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
test('evaluator complete => no blocking error, goal achieved', async () => {
|
||||
const { context, getState } = makeContext()
|
||||
const deps: GoalEvaluationDeps = {
|
||||
|
||||
@@ -5,6 +5,10 @@ import type { ToolUseContext } from '../../Tool.js'
|
||||
import type { Message } from '../../types/message.js'
|
||||
import { logForDebugging } from '../../utils/debug.js'
|
||||
import { logForDiagnosticsNoPII } from '../../utils/diagLogs.js'
|
||||
import {
|
||||
getInterruptionSignalAbortEventId,
|
||||
traceInterruptionEvent,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
import { createSystemMessage, createUserMessage } from '../../utils/messages.js'
|
||||
import { evaluateGoal as evaluateGoalDefault } from './evaluator.js'
|
||||
import { buildGoalContinuationInstruction } from './instructions.js'
|
||||
@@ -173,6 +177,12 @@ export async function* evaluateGoalAfterTurn({
|
||||
return []
|
||||
}
|
||||
|
||||
traceInterruptionEvent('goal.evaluation_started', {
|
||||
subsystem: 'goal',
|
||||
phase: 'evaluator',
|
||||
querySource,
|
||||
attemptId: goal.id,
|
||||
})
|
||||
const decision = await evaluateGoal({
|
||||
goal,
|
||||
messages: getRecentGoalEvaluationMessages(
|
||||
@@ -183,6 +193,17 @@ export async function* evaluateGoalAfterTurn({
|
||||
isNonInteractiveSession:
|
||||
toolUseContext.options.isNonInteractiveSession ?? false,
|
||||
})
|
||||
traceInterruptionEvent('goal.evaluation_completed', {
|
||||
subsystem: 'goal',
|
||||
phase: 'evaluator',
|
||||
querySource,
|
||||
attemptId: goal.id,
|
||||
outcome: decision.decision,
|
||||
reason: toolUseContext.abortController.signal.reason,
|
||||
causalEventId: getInterruptionSignalAbortEventId(
|
||||
toolUseContext.abortController.signal,
|
||||
),
|
||||
})
|
||||
|
||||
if (toolUseContext.abortController.signal.aborted) return []
|
||||
|
||||
|
||||
@@ -6,6 +6,17 @@ import {
|
||||
evaluateGoal,
|
||||
type GoalModelCaller,
|
||||
} from './evaluator.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
registerInterruptionController,
|
||||
requestAbort,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../../test/sharedMutationLock.js'
|
||||
|
||||
function user(uuid: string, content: string) {
|
||||
return {
|
||||
@@ -27,6 +38,52 @@ function assistant(uuid: string, content: string) {
|
||||
}
|
||||
|
||||
describe('goal evaluator', () => {
|
||||
test.serial('traces provider failures without serializing the error message', async () => {
|
||||
await acquireSharedMutationLock('goal/evaluator trace')
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
try {
|
||||
const controller = new AbortController()
|
||||
registerInterruptionController(controller, {
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
await evaluateGoal({
|
||||
goal: createGoalState('finish implementation'),
|
||||
messages: [],
|
||||
signal: controller.signal,
|
||||
isNonInteractiveSession: false,
|
||||
modelCaller: async () => {
|
||||
requestAbort(controller, 'user-cancel', {
|
||||
source: 'cancel_keybinding',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
throw new Error('private provider detail')
|
||||
},
|
||||
})
|
||||
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const serialized = JSON.stringify(trace)
|
||||
expect(serialized).toContain('goal.evaluation_failed')
|
||||
expect(serialized).not.toContain('private provider detail')
|
||||
const rootAbort = trace.find(entry => entry.event === 'abort.requested')
|
||||
const failed = trace.find(
|
||||
entry => entry.event === 'goal.evaluation_failed',
|
||||
)
|
||||
expect(rootAbort).toBeDefined()
|
||||
expect(failed).toBeDefined()
|
||||
expect(typeof rootAbort!.eventId).toBe('string')
|
||||
expect(typeof failed!.causalEventId).toBe('string')
|
||||
expect(failed!.causalEventId).toBe(rootAbort!.eventId)
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
test('valid complete JSON', async () => {
|
||||
const caller: GoalModelCaller = async () =>
|
||||
JSON.stringify({
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import type { SystemPrompt } from '../../utils/systemPromptType.js'
|
||||
import { asSystemPrompt } from '../../utils/systemPromptType.js'
|
||||
import {
|
||||
getInterruptionSignalAbortEventId,
|
||||
traceInterruptionEvent,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
import { queryHaiku } from '../api/claude.js'
|
||||
import type { GoalEvaluatorDecision, GoalState } from './types.js'
|
||||
|
||||
@@ -267,7 +271,14 @@ export async function evaluateGoal({
|
||||
'Goal evaluator returned malformed JSON; pausing automatic goal continuation.',
|
||||
nextInstruction: null,
|
||||
}
|
||||
} catch {
|
||||
} catch (error) {
|
||||
traceInterruptionEvent('goal.evaluation_failed', {
|
||||
subsystem: 'goal',
|
||||
phase: 'evaluator',
|
||||
error,
|
||||
reason: signal.reason,
|
||||
causalEventId: getInterruptionSignalAbortEventId(signal),
|
||||
})
|
||||
return {
|
||||
complete: false,
|
||||
confidence: 0,
|
||||
|
||||
@@ -13,6 +13,10 @@ import {
|
||||
getMissingToolResultAbortMessage,
|
||||
shouldCreateUserInterruptionMessage,
|
||||
} from '../../utils/abortReasons.js'
|
||||
import {
|
||||
getInterruptionSignalAbortEventId,
|
||||
requestAbort,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
import { runToolUse } from './toolExecution.js'
|
||||
|
||||
type MessageUpdate = {
|
||||
@@ -66,6 +70,11 @@ export class StreamingToolExecutor {
|
||||
this.toolUseContext = toolUseContext
|
||||
this.siblingAbortController = createChildAbortController(
|
||||
toolUseContext.abortController,
|
||||
undefined,
|
||||
{
|
||||
subsystem: 'streaming_tool_executor',
|
||||
controllerRole: 'sibling-tools',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -77,7 +86,11 @@ export class StreamingToolExecutor {
|
||||
discard(): void {
|
||||
if (this.discarded) return
|
||||
this.discarded = true
|
||||
this.siblingAbortController.abort('streaming_fallback')
|
||||
requestAbort(this.siblingAbortController, 'streaming_fallback', {
|
||||
source: 'streaming_fallback',
|
||||
subsystem: 'streaming_tool_executor',
|
||||
controllerRole: 'sibling-tools',
|
||||
})
|
||||
const activeLifecycleToolUseIds = new Set(
|
||||
this.toolUseContext.queryLifecycle
|
||||
?.snapshot()
|
||||
@@ -355,6 +368,11 @@ export class StreamingToolExecutor {
|
||||
// sends REJECT_MESSAGE to the model instead of aborting (#21056 regression).
|
||||
const toolAbortController = createChildAbortController(
|
||||
this.siblingAbortController,
|
||||
undefined,
|
||||
{
|
||||
subsystem: 'streaming_tool_executor',
|
||||
controllerRole: 'tool',
|
||||
},
|
||||
)
|
||||
toolAbortController.signal.addEventListener(
|
||||
'abort',
|
||||
@@ -364,8 +382,17 @@ export class StreamingToolExecutor {
|
||||
!this.toolUseContext.abortController.signal.aborted &&
|
||||
!this.discarded
|
||||
) {
|
||||
this.toolUseContext.abortController.abort(
|
||||
requestAbort(
|
||||
this.toolUseContext.abortController,
|
||||
toolAbortController.signal.reason,
|
||||
{
|
||||
source: 'tool_abort_propagation',
|
||||
causalEventId: getInterruptionSignalAbortEventId(
|
||||
toolAbortController.signal,
|
||||
),
|
||||
subsystem: 'streaming_tool_executor',
|
||||
controllerRole: 'query-root',
|
||||
},
|
||||
)
|
||||
}
|
||||
},
|
||||
@@ -414,7 +441,11 @@ export class StreamingToolExecutor {
|
||||
if (tool.block.name === BASH_TOOL_NAME) {
|
||||
this.hasErrored = true
|
||||
this.erroredToolDescription = this.getToolDescription(tool)
|
||||
this.siblingAbortController.abort('sibling_error')
|
||||
requestAbort(this.siblingAbortController, 'sibling_error', {
|
||||
source: 'sibling_error',
|
||||
subsystem: 'streaming_tool_executor',
|
||||
controllerRole: 'sibling-tools',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { afterEach, beforeEach, expect, test } from 'bun:test'
|
||||
import type { AppState } from './AppState.js'
|
||||
import { getDefaultAppState } from './AppStateStore.js'
|
||||
import { stopOrDismissAgent } from './teammateViewHelpers.js'
|
||||
import type { LocalAgentTaskState } from '../tasks/LocalAgentTask/LocalAgentTask.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
} from '../utils/interruptionTrace.js'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../test/sharedMutationLock.js'
|
||||
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock(
|
||||
'state/teammateViewHelpers.interruptionTrace.test.ts',
|
||||
)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
try {
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
} finally {
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
test('records the panel stop source and causal input before aborting an agent', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const abortController = new AbortController()
|
||||
const taskId = 'agent-task-1'
|
||||
const task = {
|
||||
id: taskId,
|
||||
type: 'local_agent',
|
||||
status: 'running',
|
||||
description: 'test agent',
|
||||
startTime: Date.now(),
|
||||
outputFile: '/tmp/test-agent-output',
|
||||
outputOffset: 0,
|
||||
notified: false,
|
||||
agentId: 'agent-1',
|
||||
prompt: 'test',
|
||||
agentType: 'general-purpose',
|
||||
abortController,
|
||||
retrieved: false,
|
||||
lastReportedToolCount: 0,
|
||||
lastReportedTokenCount: 0,
|
||||
isBackgrounded: true,
|
||||
pendingMessages: [],
|
||||
retain: false,
|
||||
diskLoaded: false,
|
||||
} satisfies LocalAgentTaskState
|
||||
let state: AppState = {
|
||||
...getDefaultAppState(),
|
||||
tasks: { [taskId]: task },
|
||||
}
|
||||
|
||||
stopOrDismissAgent(taskId, updater => {
|
||||
state = updater(state)
|
||||
})
|
||||
|
||||
const entries = __getInterruptionTraceSnapshotForTests()
|
||||
const input = entries.find(entry => entry.event === 'input.agent_panel_stop')
|
||||
const requested = entries.find(entry => entry.event === 'abort.requested')
|
||||
expect(abortController.signal.aborted).toBe(true)
|
||||
expect(input).toBeDefined()
|
||||
expect(requested).toMatchObject({
|
||||
source: 'agent_panel_stop',
|
||||
subsystem: 'local_agent_task',
|
||||
controllerRole: 'background-agent',
|
||||
subagentId: taskId,
|
||||
causalEventId: input?.eventId,
|
||||
})
|
||||
})
|
||||
@@ -1,6 +1,10 @@
|
||||
import { logEvent } from '../services/analytics/index.js'
|
||||
import { isTerminalTaskStatus } from '../Task.js'
|
||||
import type { LocalAgentTaskState } from '../tasks/LocalAgentTask/LocalAgentTask.js'
|
||||
import {
|
||||
requestAbort,
|
||||
traceInterruptionEvent,
|
||||
} from '../utils/interruptionTrace.js'
|
||||
|
||||
// Inlined from framework.ts — importing creates a cycle through
|
||||
// BackgroundTasksDialog. Keep in sync with PANEL_GRACE_MS there.
|
||||
@@ -121,7 +125,20 @@ export function stopOrDismissAgent(
|
||||
const task = prev.tasks[taskId]
|
||||
if (!isLocalAgent(task)) return prev
|
||||
if (task.status === 'running') {
|
||||
task.abortController?.abort()
|
||||
const causalEventId = traceInterruptionEvent('input.agent_panel_stop', {
|
||||
source: 'agent_panel_stop',
|
||||
subsystem: 'local_agent_task',
|
||||
subagentId: taskId,
|
||||
})
|
||||
if (task.abortController) {
|
||||
requestAbort(task.abortController, undefined, {
|
||||
source: 'agent_panel_stop',
|
||||
subsystem: 'local_agent_task',
|
||||
controllerRole: 'background-agent',
|
||||
subagentId: taskId,
|
||||
causalEventId,
|
||||
})
|
||||
}
|
||||
return prev
|
||||
}
|
||||
if (task.evictAfter === 0) return prev
|
||||
|
||||
@@ -16,6 +16,7 @@ import { registerCleanup } from '../../utils/cleanupRegistry.js';
|
||||
import { getToolSearchOrReadInfo } from '../../utils/collapseReadSearch.js';
|
||||
import { enqueuePendingNotification } from '../../utils/messageQueueManager.js';
|
||||
import { getAgentTranscriptPath } from '../../utils/sessionStorage.js';
|
||||
import { requestAbort } from '../../utils/interruptionTrace.js';
|
||||
import { evictTaskOutput, getTaskOutputPath, initTaskOutputAsSymlink } from '../../utils/task/diskOutput.js';
|
||||
import { PANEL_GRACE_MS, registerTask, updateTaskState } from '../../utils/task/framework.js';
|
||||
import { emitTaskProgress } from '../../utils/task/sdkProgress.js';
|
||||
@@ -271,21 +272,34 @@ export const LocalAgentTask: Task = {
|
||||
name: 'LocalAgentTask',
|
||||
type: 'local_agent',
|
||||
async kill(taskId, setAppState) {
|
||||
killAsyncAgent(taskId, setAppState);
|
||||
killAsyncAgent(taskId, setAppState, { source: 'task_stop' });
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Kill an agent task. No-op if already killed/completed.
|
||||
*/
|
||||
export function killAsyncAgent(taskId: string, setAppState: SetAppState): void {
|
||||
export type AgentKillTrace = {
|
||||
source: string;
|
||||
causalEventId?: string;
|
||||
};
|
||||
|
||||
export function killAsyncAgent(taskId: string, setAppState: SetAppState, trace: AgentKillTrace = { source: 'agent_cleanup' }): void {
|
||||
let killed = false;
|
||||
updateTaskState<LocalAgentTaskState>(taskId, setAppState, task => {
|
||||
if (task.status !== 'running') {
|
||||
return task;
|
||||
}
|
||||
killed = true;
|
||||
task.abortController?.abort();
|
||||
if (task.abortController && !task.abortController.signal.aborted) {
|
||||
requestAbort(task.abortController, undefined, {
|
||||
source: trace.source,
|
||||
subsystem: 'local_agent_task',
|
||||
controllerRole: 'background-agent',
|
||||
subagentId: taskId,
|
||||
causalEventId: trace.causalEventId
|
||||
});
|
||||
}
|
||||
task.unregisterCleanup?.();
|
||||
return {
|
||||
...task,
|
||||
@@ -306,10 +320,10 @@ export function killAsyncAgent(taskId: string, setAppState: SetAppState): void {
|
||||
* Kill all running agent tasks.
|
||||
* Used by ESC cancellation in coordinator mode to stop all subagents.
|
||||
*/
|
||||
export function killAllRunningAgentTasks(tasks: Record<string, TaskState>, setAppState: SetAppState): void {
|
||||
export function killAllRunningAgentTasks(tasks: Record<string, TaskState>, setAppState: SetAppState, trace: AgentKillTrace): void {
|
||||
for (const [taskId, task] of Object.entries(tasks)) {
|
||||
if (task.type === 'local_agent' && task.status === 'running') {
|
||||
killAsyncAgent(taskId, setAppState);
|
||||
killAsyncAgent(taskId, setAppState, trace);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -483,7 +497,13 @@ export function registerAsyncAgent({
|
||||
void initTaskOutputAsSymlink(agentId, getAgentTranscriptPath(asAgentId(agentId)));
|
||||
|
||||
// Create abort controller - if parent provided, create child that auto-aborts with parent
|
||||
const abortController = parentAbortController ? createChildAbortController(parentAbortController) : createAbortController();
|
||||
const abortController = parentAbortController
|
||||
? createChildAbortController(parentAbortController, undefined, {
|
||||
subsystem: 'local_agent_task',
|
||||
controllerRole: 'background-agent',
|
||||
subagentId: agentId,
|
||||
})
|
||||
: createAbortController();
|
||||
const taskState: LocalAgentTaskState = {
|
||||
...createTaskStateBase(agentId, 'local_agent', description, toolUseId),
|
||||
type: 'local_agent',
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { getFsImplementation, setFsImplementation } from '../../utils/fsOperations.js'
|
||||
import { gracefulShutdown, resetShutdownState } from '../../utils/gracefulShutdown.js'
|
||||
import {
|
||||
__resetInterruptionTraceForTests,
|
||||
requestAbort,
|
||||
traceInterruptionEvent,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
|
||||
process.env.NODE_ENV = 'test'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/virtual/trace.jsonl'
|
||||
|
||||
const originalFs = getFsImplementation()
|
||||
const originalExit = process.exit
|
||||
let writeSettled = false
|
||||
let exitObservedSettledWrite = false
|
||||
let exitCalled = false
|
||||
|
||||
setFsImplementation({
|
||||
...originalFs,
|
||||
mkdirSync: () => {},
|
||||
appendRegularFile: async () => {
|
||||
if (process.env.TRACE_SHUTDOWN_BLOCK_WRITE === '1') {
|
||||
await new Promise<void>(() => {})
|
||||
}
|
||||
await Bun.sleep(50)
|
||||
writeSettled = true
|
||||
},
|
||||
})
|
||||
process.exit = ((_code?: number) => {
|
||||
exitCalled = true
|
||||
exitObservedSettledWrite = writeSettled
|
||||
throw new Error('mocked process exit')
|
||||
}) as typeof process.exit
|
||||
|
||||
try {
|
||||
const controller = new AbortController()
|
||||
if (process.env.TRACE_SHUTDOWN_PENDING_ONLY === '1') {
|
||||
controller.abort('interrupt')
|
||||
traceInterruptionEvent('shutdown.pending')
|
||||
} else {
|
||||
requestAbort(controller, 'interrupt', {
|
||||
source: 'print_sigint',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
await gracefulShutdown(0).catch(() => {})
|
||||
console.log(
|
||||
`TRACE_SHUTDOWN_RESULT ${JSON.stringify({
|
||||
aborted: controller.signal.aborted,
|
||||
exitCalled,
|
||||
writeSettled,
|
||||
exitObservedSettledWrite,
|
||||
})}`,
|
||||
)
|
||||
} finally {
|
||||
process.exit = originalExit
|
||||
setFsImplementation(originalFs)
|
||||
resetShutdownState()
|
||||
__resetInterruptionTraceForTests()
|
||||
}
|
||||
@@ -40,6 +40,7 @@ import {
|
||||
} from '../../utils/teammateMailbox.js'
|
||||
import { resumeAgentBackground } from '../AgentTool/resumeAgent.js'
|
||||
import { SEND_MESSAGE_TOOL_NAME } from './constants.js'
|
||||
import { abortApprovedInProcessTeammate } from './shutdownInterruptionTrace.js'
|
||||
import { DESCRIPTION, getPrompt } from './prompt.js'
|
||||
import { renderToolResultMessage, renderToolUseMessage } from './UI.js'
|
||||
|
||||
@@ -354,7 +355,9 @@ async function handleShutdownApproval(
|
||||
const appState = context.getAppState()
|
||||
const task = findTeammateTaskByAgentId(agentId, appState.tasks)
|
||||
if (task?.abortController) {
|
||||
task.abortController.abort()
|
||||
abortApprovedInProcessTeammate(task.abortController, {
|
||||
agentId,
|
||||
})
|
||||
logForDebugging(
|
||||
`[SendMessageTool] Aborted controller for in-process teammate ${agentName}`,
|
||||
)
|
||||
@@ -372,7 +375,9 @@ async function handleShutdownApproval(
|
||||
logForDebugging(
|
||||
`[SendMessageTool] Fallback: Found in-process task for ${agentName} via AppState, aborting`,
|
||||
)
|
||||
task.abortController.abort()
|
||||
abortApprovedInProcessTeammate(task.abortController, {
|
||||
agentId,
|
||||
})
|
||||
|
||||
return {
|
||||
data: {
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { afterEach, beforeEach, expect, test } from 'bun:test'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../../test/sharedMutationLock.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
registerInterruptionController,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
import { abortApprovedInProcessTeammate } from './shutdownInterruptionTrace.js'
|
||||
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock('shutdownInterruptionTrace.test.ts')
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
} finally {
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
test('shutdown approval records its input before requesting the teammate abort', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const controller = new AbortController()
|
||||
registerInterruptionController(controller, {
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: 'agent-1',
|
||||
})
|
||||
|
||||
abortApprovedInProcessTeammate(controller, {
|
||||
agentId: 'agent-1',
|
||||
})
|
||||
|
||||
expect(controller.signal.aborted).toBe(true)
|
||||
expect(controller.signal.reason).toBeInstanceOf(DOMException)
|
||||
expect((controller.signal.reason as DOMException).name).toBe('AbortError')
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const approved = trace.find(
|
||||
entry => entry.event === 'teammate.shutdown_approved',
|
||||
)
|
||||
const requested = trace.find(entry => entry.event === 'abort.requested')
|
||||
expect(approved).toMatchObject({
|
||||
source: 'shutdown_approved',
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: 'agent-1',
|
||||
})
|
||||
expect(requested).toMatchObject({
|
||||
source: 'shutdown_approved',
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: 'agent-1',
|
||||
causalEventId: approved?.eventId,
|
||||
})
|
||||
expect(trace.indexOf(approved!)).toBeLessThan(trace.indexOf(requested!))
|
||||
})
|
||||
@@ -0,0 +1,23 @@
|
||||
import {
|
||||
requestAbort,
|
||||
traceInterruptionEvent,
|
||||
} from '../../utils/interruptionTrace.js'
|
||||
|
||||
export function abortApprovedInProcessTeammate(
|
||||
controller: AbortController,
|
||||
identity: { agentId: string },
|
||||
): void {
|
||||
const causalEventId = traceInterruptionEvent('teammate.shutdown_approved', {
|
||||
source: 'shutdown_approved',
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: identity.agentId,
|
||||
})
|
||||
requestAbort(controller, undefined, {
|
||||
source: 'shutdown_approved',
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: identity.agentId,
|
||||
causalEventId,
|
||||
})
|
||||
}
|
||||
+91
-1
@@ -29,6 +29,10 @@
|
||||
* )
|
||||
*/
|
||||
import { createSignal } from './signal.js'
|
||||
import {
|
||||
flushInterruptionTrace,
|
||||
traceInterruptionEvent,
|
||||
} from './interruptionTrace.js'
|
||||
import type {
|
||||
QueryActiveOperationSnapshot,
|
||||
QueryGuardMetadata,
|
||||
@@ -127,6 +131,8 @@ export class QueryGuard {
|
||||
private _suspendedAt = 0
|
||||
private _totalSuspendedMs = 0
|
||||
private _leaseCounter = 0
|
||||
private _traceActivityCount = 0
|
||||
private _lastTraceActivityAt = 0
|
||||
private _activeLeases = new Map<string, LeaseRecord>()
|
||||
private _context: QueryLifecycleContext | null = null
|
||||
private _lastContext: QueryLifecycleContext | null = null
|
||||
@@ -190,8 +196,17 @@ export class QueryGuard {
|
||||
this._totalSuspendedMs = 0
|
||||
this._lastContext = null
|
||||
this._queryStartedAt = performance.now()
|
||||
this._traceActivityCount = 0
|
||||
this._lastTraceActivityAt = 0
|
||||
this._lastActivityAt = this._queryStartedAt
|
||||
this._context = this._createContext(metadata)
|
||||
traceInterruptionEvent('query_guard.started', {
|
||||
subsystem: 'query_guard',
|
||||
phase: 'running',
|
||||
queryId: this._context.queryId,
|
||||
queryGeneration: this._generation,
|
||||
querySource: this._context.querySource,
|
||||
})
|
||||
this._getActiveOperations = metadata?.getActiveOperations ?? null
|
||||
this._startTimeout()
|
||||
this._notify()
|
||||
@@ -221,7 +236,16 @@ export class QueryGuard {
|
||||
this._suspendCount = 0
|
||||
this._suspendedAt = 0
|
||||
this._totalSuspendedMs = 0
|
||||
const context = this._context
|
||||
this._completeContext(terminalReason, abortReason)
|
||||
traceInterruptionEvent('query_guard.ended', {
|
||||
subsystem: 'query_guard',
|
||||
phase: terminalReason,
|
||||
queryId: context?.queryId,
|
||||
queryGeneration: generation,
|
||||
querySource: context?.querySource,
|
||||
reason: abortReason,
|
||||
})
|
||||
this._status = 'idle'
|
||||
this._getActiveOperations = null
|
||||
this._notify()
|
||||
@@ -244,7 +268,16 @@ export class QueryGuard {
|
||||
this._suspendCount = 0
|
||||
this._suspendedAt = 0
|
||||
this._totalSuspendedMs = 0
|
||||
const context = this._context
|
||||
this._completeContext(terminalReason, abortReason)
|
||||
traceInterruptionEvent('query_guard.force_ended', {
|
||||
subsystem: 'query_guard',
|
||||
phase: terminalReason,
|
||||
queryId: context?.queryId,
|
||||
queryGeneration: this._generation,
|
||||
querySource: context?.querySource,
|
||||
reason: abortReason,
|
||||
})
|
||||
this._status = 'idle'
|
||||
this._getActiveOperations = null
|
||||
++this._generation
|
||||
@@ -260,7 +293,22 @@ export class QueryGuard {
|
||||
void reason
|
||||
if (this._status !== 'running') return
|
||||
if (generation !== undefined && generation !== this._generation) return
|
||||
this._lastActivityAt = performance.now()
|
||||
const now = performance.now()
|
||||
this._lastActivityAt = now
|
||||
this._traceActivityCount++
|
||||
if (
|
||||
this._traceActivityCount === 1 ||
|
||||
now - this._lastTraceActivityAt >= 5_000
|
||||
) {
|
||||
this._lastTraceActivityAt = now
|
||||
traceInterruptionEvent('query_guard.activity', {
|
||||
subsystem: 'query_guard',
|
||||
queryId: this._context?.queryId,
|
||||
queryGeneration: this._generation,
|
||||
querySource: this._context?.querySource,
|
||||
yieldedEventCount: this._traceActivityCount,
|
||||
})
|
||||
}
|
||||
this._scheduleTimeout()
|
||||
}
|
||||
|
||||
@@ -314,6 +362,13 @@ export class QueryGuard {
|
||||
deadlineAt: leaseDeadlineAt,
|
||||
description: input.description,
|
||||
})
|
||||
traceInterruptionEvent('query_guard.lease_acquired', {
|
||||
subsystem: 'query_guard',
|
||||
queryId: this._context?.queryId,
|
||||
queryGeneration: this._generation,
|
||||
source: input.owner,
|
||||
leaseCount: this._activeLeases.size,
|
||||
})
|
||||
this._lastActivityAt = now
|
||||
this._scheduleTimeout()
|
||||
|
||||
@@ -331,6 +386,13 @@ export class QueryGuard {
|
||||
const lease = this._activeLeases.get(leaseId)
|
||||
if (!lease || lease.generation !== generation) return
|
||||
this._activeLeases.delete(leaseId)
|
||||
traceInterruptionEvent('query_guard.lease_released', {
|
||||
subsystem: 'query_guard',
|
||||
queryId: this._context?.queryId,
|
||||
queryGeneration: this._generation,
|
||||
source: lease.owner,
|
||||
leaseCount: this._activeLeases.size,
|
||||
})
|
||||
this._scheduleTimeout()
|
||||
}
|
||||
|
||||
@@ -352,6 +414,12 @@ export class QueryGuard {
|
||||
this._suspendedAt = performance.now()
|
||||
}
|
||||
this._suspendCount++
|
||||
traceInterruptionEvent('query_guard.suspended', {
|
||||
subsystem: 'query_guard',
|
||||
queryId: this._context?.queryId,
|
||||
queryGeneration: this._generation,
|
||||
suspendCount: this._suspendCount,
|
||||
})
|
||||
this._scheduleTimeout()
|
||||
|
||||
let resumed = false
|
||||
@@ -362,6 +430,12 @@ export class QueryGuard {
|
||||
if (generation !== this._generation) return
|
||||
if (this._suspendCount === 0) return
|
||||
this._suspendCount--
|
||||
traceInterruptionEvent('query_guard.resumed', {
|
||||
subsystem: 'query_guard',
|
||||
queryId: this._context?.queryId,
|
||||
queryGeneration: this._generation,
|
||||
suspendCount: this._suspendCount,
|
||||
})
|
||||
if (this._suspendCount === 0) {
|
||||
this._resumeAfterInteraction()
|
||||
}
|
||||
@@ -534,6 +608,21 @@ export class QueryGuard {
|
||||
context,
|
||||
activeOperations: this._snapshotActiveOperations(),
|
||||
}
|
||||
const causalEventId = traceInterruptionEvent('query_guard.fired', {
|
||||
subsystem: 'query_guard',
|
||||
phase: terminalReason,
|
||||
queryId: context.queryId,
|
||||
queryGeneration: context.queryGeneration,
|
||||
querySource: context.querySource,
|
||||
trigger: reason,
|
||||
elapsedQueryMs: timeout.elapsedMs,
|
||||
sinceLastActivityMs: now - this._lastActivityAt,
|
||||
activeApiCallCount: timeout.activeOperations.apiCalls.length,
|
||||
activeToolUseCount: timeout.activeOperations.toolUses.length,
|
||||
leaseCount: this._activeLeases.size,
|
||||
suspendCount: this._suspendCount,
|
||||
})
|
||||
if (causalEventId) timeout.causalEventId = causalEventId
|
||||
|
||||
console.error(
|
||||
`[QueryGuard] Query ${reason} timeout - force-ending to prevent infinite spinner`,
|
||||
@@ -544,6 +633,7 @@ export class QueryGuard {
|
||||
console.error('[QueryGuard] Timeout handler failed', error)
|
||||
} finally {
|
||||
this.forceEnd(terminalReason, reason)
|
||||
flushInterruptionTrace('query_guard_timeout')
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
import { setMaxListeners } from 'events'
|
||||
import {
|
||||
getInterruptionSignalAbortEventId,
|
||||
getInterruptionSignalId,
|
||||
registerInterruptionController,
|
||||
requestAbort,
|
||||
type InterruptionTraceFields,
|
||||
} from './interruptionTrace.js'
|
||||
|
||||
/**
|
||||
* Default max listeners for standard operations
|
||||
@@ -32,7 +39,19 @@ function propagateAbort(
|
||||
weakChild: WeakRef<AbortController>,
|
||||
): void {
|
||||
const parent = this.deref()
|
||||
weakChild.deref()?.abort(parent?.signal.reason)
|
||||
const child = weakChild.deref()
|
||||
if (!child) return
|
||||
requestAbort(child, parent?.signal.reason, {
|
||||
source: 'parent_signal',
|
||||
subsystem: 'abort_controller',
|
||||
controllerRole: 'child',
|
||||
...(parent && {
|
||||
causalEventId: getInterruptionSignalAbortEventId(parent.signal),
|
||||
}),
|
||||
...(parent && {
|
||||
parentControllerIds: [getInterruptionSignalId(parent.signal) ?? 'unregistered-parent'],
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -63,17 +82,40 @@ function removeAbortHandler(
|
||||
*
|
||||
* @param parent - The parent AbortController
|
||||
* @param maxListeners - Maximum number of listeners (default: 50)
|
||||
* @param traceFields - Owning lifecycle identity for interruption diagnostics
|
||||
* @returns Child AbortController
|
||||
*/
|
||||
export function createChildAbortController(
|
||||
parent: AbortController,
|
||||
maxListeners?: number,
|
||||
traceFields: InterruptionTraceFields = {},
|
||||
): AbortController {
|
||||
const child = createAbortController(maxListeners)
|
||||
const parentId = registerInterruptionController(parent, {
|
||||
subsystem: 'abort_controller',
|
||||
controllerRole: 'parent',
|
||||
}, { provisionalRole: true })
|
||||
const hasOwningRole = traceFields.controllerRole !== undefined
|
||||
registerInterruptionController(
|
||||
child,
|
||||
{
|
||||
subsystem: 'abort_controller',
|
||||
controllerRole: 'child',
|
||||
...traceFields,
|
||||
...(parentId && { parentControllerIds: [parentId] }),
|
||||
},
|
||||
{ provisionalRole: !hasOwningRole },
|
||||
)
|
||||
|
||||
// Fast path: parent already aborted, no listener setup needed
|
||||
if (parent.signal.aborted) {
|
||||
child.abort(parent.signal.reason)
|
||||
requestAbort(child, parent.signal.reason, {
|
||||
source: 'already_aborted_parent',
|
||||
subsystem: 'abort_controller',
|
||||
controllerRole: 'child',
|
||||
causalEventId: getInterruptionSignalAbortEventId(parent.signal),
|
||||
...(parentId && { parentControllerIds: [parentId] }),
|
||||
})
|
||||
return child
|
||||
}
|
||||
|
||||
|
||||
@@ -2699,7 +2699,14 @@ export function startRelevantMemoryPrefetch(
|
||||
|
||||
// Chained to the turn-level abort so user Escape cancels the sideQuery
|
||||
// immediately, not just on [Symbol.dispose] when queryLoop exits.
|
||||
const controller = createChildAbortController(toolUseContext.abortController)
|
||||
const controller = createChildAbortController(
|
||||
toolUseContext.abortController,
|
||||
undefined,
|
||||
{
|
||||
subsystem: 'memory_attachments',
|
||||
controllerRole: 'memory-prefetch',
|
||||
},
|
||||
)
|
||||
const firedAt = Date.now()
|
||||
const promise = getRelevantMemoryAttachments(
|
||||
input,
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
import { createAbortController } from './abortController.js'
|
||||
import {
|
||||
getInterruptionSignalAbortEventId,
|
||||
getInterruptionSignalId,
|
||||
registerInterruptionSignal,
|
||||
requestAbort,
|
||||
traceCombinedSignal,
|
||||
traceInterruptionEvent,
|
||||
} from './interruptionTrace.js'
|
||||
|
||||
/**
|
||||
* Creates a combined AbortSignal that aborts when the input signal aborts,
|
||||
@@ -14,17 +22,47 @@ import { createAbortController } from './abortController.js'
|
||||
*/
|
||||
export function createCombinedAbortSignal(
|
||||
signal: AbortSignal | undefined,
|
||||
opts?: { signalB?: AbortSignal; timeoutMs?: number },
|
||||
opts?: {
|
||||
signalB?: AbortSignal
|
||||
timeoutMs?: number
|
||||
trace?: { subsystem: string; controllerRole?: string }
|
||||
},
|
||||
): { signal: AbortSignal; cleanup: () => void } {
|
||||
const { signalB, timeoutMs } = opts ?? {}
|
||||
const { signalB, timeoutMs, trace } = opts ?? {}
|
||||
const combined = createAbortController()
|
||||
const traceFields = {
|
||||
subsystem: trace?.subsystem ?? 'combined_abort_signal',
|
||||
controllerRole: trace?.controllerRole ?? 'combined',
|
||||
}
|
||||
const parentIds = [signal, signalB]
|
||||
.filter((parent): parent is AbortSignal => parent !== undefined)
|
||||
.map(parent =>
|
||||
registerInterruptionSignal(parent, {
|
||||
...traceFields,
|
||||
controllerRole: 'combined-parent',
|
||||
}),
|
||||
)
|
||||
.filter((id): id is string => id !== undefined)
|
||||
traceCombinedSignal(combined, [signal, signalB], traceFields)
|
||||
|
||||
if (signal?.aborted) {
|
||||
combined.abort(signal.reason)
|
||||
requestAbort(combined, signal.reason, {
|
||||
...traceFields,
|
||||
source: 'signal_a_already_aborted',
|
||||
parentControllerIds: parentIds,
|
||||
winningParentControllerId: getInterruptionSignalId(signal),
|
||||
causalEventId: getInterruptionSignalAbortEventId(signal),
|
||||
})
|
||||
return { signal: combined.signal, cleanup: () => {} }
|
||||
}
|
||||
if (signalB?.aborted) {
|
||||
combined.abort(signalB.reason)
|
||||
requestAbort(combined, signalB.reason, {
|
||||
...traceFields,
|
||||
source: 'signal_b_already_aborted',
|
||||
parentControllerIds: parentIds,
|
||||
winningParentControllerId: getInterruptionSignalId(signalB),
|
||||
causalEventId: getInterruptionSignalAbortEventId(signalB),
|
||||
})
|
||||
return { signal: combined.signal, cleanup: () => {} }
|
||||
}
|
||||
|
||||
@@ -39,15 +77,35 @@ export function createCombinedAbortSignal(
|
||||
}
|
||||
signal?.removeEventListener('abort', abortFromSignal)
|
||||
signalB?.removeEventListener('abort', abortFromSignalB)
|
||||
traceInterruptionEvent('combined_signal.cleanup', traceFields)
|
||||
}
|
||||
const abortCombined = (reason?: unknown) => {
|
||||
const abortCombined = (
|
||||
reason: unknown,
|
||||
source: string,
|
||||
winningParent?: AbortSignal,
|
||||
) => {
|
||||
cleanup()
|
||||
combined.abort(reason)
|
||||
requestAbort(combined, reason, {
|
||||
...traceFields,
|
||||
source,
|
||||
parentControllerIds: parentIds,
|
||||
winningParentControllerId: winningParent
|
||||
? getInterruptionSignalId(winningParent)
|
||||
: undefined,
|
||||
causalEventId: winningParent
|
||||
? getInterruptionSignalAbortEventId(winningParent)
|
||||
: undefined,
|
||||
})
|
||||
}
|
||||
const abortFromSignal = () => abortCombined(signal?.reason)
|
||||
const abortFromSignalB = () => abortCombined(signalB?.reason)
|
||||
const abortFromSignal = () =>
|
||||
abortCombined(signal?.reason, 'signal_a', signal)
|
||||
const abortFromSignalB = () =>
|
||||
abortCombined(signalB?.reason, 'signal_b', signalB)
|
||||
const abortFromTimeout = () =>
|
||||
abortCombined(new DOMException('The operation timed out.', 'TimeoutError'))
|
||||
abortCombined(
|
||||
new DOMException('The operation timed out.', 'TimeoutError'),
|
||||
'timeout',
|
||||
)
|
||||
|
||||
if (timeoutMs !== undefined) {
|
||||
timer = setTimeout(abortFromTimeout, timeoutMs)
|
||||
|
||||
@@ -22,6 +22,7 @@ import { getSessionId } from '../../bootstrap/state.js';
|
||||
import { ComputerUseApproval } from '../../components/permissions/ComputerUseApproval/ComputerUseApproval.js';
|
||||
import type { Tool, ToolUseContext } from '../../Tool.js';
|
||||
import { logForDebugging } from '../debug.js';
|
||||
import { requestAbort } from '../interruptionTrace.js';
|
||||
import { checkComputerUseLock, tryAcquireComputerUseLock } from './computerUseLock.js';
|
||||
import { registerEscHotkey } from './escHotkey.js';
|
||||
import { getChicagoCoordinateMode } from './gates.js';
|
||||
@@ -219,7 +220,11 @@ export function buildSessionContext(): ComputerUseSessionContext {
|
||||
// holds a pump retain until unregisterEscHotkey() in cleanup.ts.
|
||||
const escRegistered = registerEscHotkey(() => {
|
||||
logForDebugging('[cu-esc] user escape, aborting turn');
|
||||
tuc().abortController.abort();
|
||||
requestAbort(tuc().abortController, undefined, {
|
||||
source: 'computer_use_escape',
|
||||
subsystem: 'computer_use',
|
||||
controllerRole: 'tool',
|
||||
});
|
||||
});
|
||||
tuc().sendOSNotification?.({
|
||||
message: escRegistered ? 'Claude is using your computer · press Esc to stop' : 'Claude is using your computer · press Ctrl+C to stop',
|
||||
|
||||
+37
-2
@@ -1,4 +1,5 @@
|
||||
import { dirname } from 'path'
|
||||
import { getErrnoCode } from './errors.js'
|
||||
import { getFsImplementation } from './fsOperations.js'
|
||||
import { jsonStringify } from './slowOperations.js'
|
||||
|
||||
@@ -11,6 +12,40 @@ type DiagnosticLogEntry = {
|
||||
data: Record<string, unknown>
|
||||
}
|
||||
|
||||
export type DiagnosticAppendResult =
|
||||
| 'committed'
|
||||
| 'unsupported'
|
||||
| 'retryable_failure'
|
||||
| 'uncertain_failure'
|
||||
|
||||
/**
|
||||
* Append already-sanitized diagnostic records to an explicit file.
|
||||
*
|
||||
* This is the shared filesystem boundary for opt-in diagnostic streams. Callers
|
||||
* must rebuild records from an allowlist before using it; arbitrary runtime
|
||||
* objects, messages, prompts, and file paths are not safe inputs.
|
||||
*/
|
||||
export async function appendDiagnosticsNoPII(
|
||||
logFile: string,
|
||||
entries: readonly Record<string, unknown>[],
|
||||
): Promise<DiagnosticAppendResult> {
|
||||
if (entries.length === 0) return 'committed'
|
||||
// Node exposes the descriptor-relative namespace required by the secure
|
||||
// append implementation through /proc/self/fd on Linux only.
|
||||
if (process.platform !== 'linux') return 'unsupported'
|
||||
|
||||
try {
|
||||
const fs = getFsImplementation()
|
||||
const lines = entries.map(entry => jsonStringify(entry)).join('\n') + '\n'
|
||||
await fs.appendRegularFile(logFile, lines, { mode: 0o600 })
|
||||
return 'committed'
|
||||
} catch (error) {
|
||||
return getErrnoCode(error) === 'ERR_DIAGNOSTIC_APPEND_UNCERTAIN'
|
||||
? 'uncertain_failure'
|
||||
: 'retryable_failure'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs diagnostic information to a logfile. This information is sent
|
||||
* via the environment manager to session-ingress to monitor issues from
|
||||
@@ -46,12 +81,12 @@ export function logForDiagnosticsNoPII(
|
||||
try {
|
||||
fs.appendFileSync(logFile, line)
|
||||
} catch {
|
||||
// If append fails, try creating the directory first
|
||||
// Preserve the legacy diagnostic logger's append-through-symlink contract.
|
||||
try {
|
||||
fs.mkdirSync(dirname(logFile))
|
||||
fs.appendFileSync(logFile, line)
|
||||
} catch {
|
||||
// Silently fail if logging is not possible
|
||||
// Silently fail if logging is not possible.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -358,7 +358,11 @@ export function createSubagentContext(
|
||||
overrides?.abortController ??
|
||||
(overrides?.shareAbortController
|
||||
? parentContext.abortController
|
||||
: createChildAbortController(parentContext.abortController))
|
||||
: createChildAbortController(parentContext.abortController, undefined, {
|
||||
subsystem: 'forked_agent',
|
||||
controllerRole: 'subagent-context',
|
||||
subagentId: overrides?.agentId,
|
||||
}))
|
||||
|
||||
// Determine getAppState - wrap to set shouldAvoidPermissionPrompts unless sharing abortController
|
||||
// (if sharing abortController, it's an interactive agent that CAN show UI)
|
||||
|
||||
@@ -97,6 +97,18 @@ export type FsOperations = {
|
||||
}
|
||||
/** Appends string to file */
|
||||
appendFileSync(path: string, data: string, options?: { mode?: number }): void
|
||||
/**
|
||||
* Opens a regular file without following path symlinks and appends data.
|
||||
*
|
||||
* Linux only and requires an absolute path; throws on unsupported platforms
|
||||
* or relative paths. A partial write that cannot be rolled back rejects with
|
||||
* `ERR_DIAGNOSTIC_APPEND_UNCERTAIN`.
|
||||
*/
|
||||
appendRegularFile(
|
||||
path: string,
|
||||
data: string,
|
||||
options?: { mode?: number },
|
||||
): Promise<void>
|
||||
/** Copies file from source to destination */
|
||||
copyFileSync(src: string, dest: string): void
|
||||
/** Deletes file */
|
||||
@@ -546,6 +558,103 @@ export const NodeFsOperations: FsOperations = {
|
||||
fs.appendFileSync(path, data)
|
||||
},
|
||||
|
||||
async appendRegularFile(path, data, options) {
|
||||
if (process.platform !== 'linux') {
|
||||
throw new Error('Secure diagnostic file output is available on Linux only')
|
||||
}
|
||||
if (!nodePath.isAbsolute(path)) {
|
||||
throw new Error('Secure diagnostic file output requires an absolute path')
|
||||
}
|
||||
|
||||
const resolvedPath = nodePath.resolve(path)
|
||||
const components = nodePath
|
||||
.relative(nodePath.parse(resolvedPath).root, nodePath.dirname(resolvedPath))
|
||||
.split(nodePath.sep)
|
||||
.filter(Boolean)
|
||||
const directoryFlags = fs.constants.O_RDONLY |
|
||||
fs.constants.O_DIRECTORY |
|
||||
fs.constants.O_NOFOLLOW
|
||||
const descriptorDirectory = '/proc/self/fd'
|
||||
let directoryHandle = await open('/', directoryFlags)
|
||||
let committed = false
|
||||
let operationError: unknown
|
||||
|
||||
try {
|
||||
for (const component of components) {
|
||||
const descriptorPath = `${descriptorDirectory}/${directoryHandle.fd}/${component}`
|
||||
let nextDirectoryHandle
|
||||
try {
|
||||
nextDirectoryHandle = await open(descriptorPath, directoryFlags)
|
||||
} catch (error) {
|
||||
if (getErrnoCode(error) !== 'ENOENT') throw error
|
||||
await mkdirPromise(descriptorPath, { mode: 0o700 })
|
||||
nextDirectoryHandle = await open(descriptorPath, directoryFlags)
|
||||
}
|
||||
try {
|
||||
await directoryHandle.close()
|
||||
} catch (error) {
|
||||
await nextDirectoryHandle.close().catch(() => {})
|
||||
throw error
|
||||
}
|
||||
directoryHandle = nextDirectoryHandle
|
||||
}
|
||||
|
||||
const descriptorPath = `${descriptorDirectory}/${directoryHandle.fd}/${nodePath.basename(resolvedPath)}`
|
||||
const flags = fs.constants.O_APPEND |
|
||||
fs.constants.O_CREAT |
|
||||
fs.constants.O_NONBLOCK |
|
||||
fs.constants.O_WRONLY |
|
||||
fs.constants.O_NOFOLLOW
|
||||
const handle = await open(descriptorPath, flags, options?.mode ?? 0o600)
|
||||
let fileOperationError: unknown
|
||||
try {
|
||||
const stats = await handle.stat()
|
||||
if (!stats.isFile()) {
|
||||
throw new Error('Diagnostics target is not a regular file')
|
||||
}
|
||||
if (options?.mode !== undefined) await handle.chmod(options.mode)
|
||||
try {
|
||||
await handle.writeFile(data, { encoding: 'utf8' })
|
||||
committed = true
|
||||
} catch (error) {
|
||||
try {
|
||||
await handle.truncate(stats.size)
|
||||
} catch (rollbackError) {
|
||||
throw Object.assign(
|
||||
new Error('Diagnostic append may have partially committed', {
|
||||
cause: rollbackError,
|
||||
}),
|
||||
{ code: 'ERR_DIAGNOSTIC_APPEND_UNCERTAIN' },
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
} catch (error) {
|
||||
fileOperationError = error
|
||||
}
|
||||
try {
|
||||
await handle.close()
|
||||
} catch (error) {
|
||||
// A close failure after writeFile completed cannot safely be interpreted
|
||||
// as a failed append; replaying would duplicate the committed batch.
|
||||
if (!committed && fileOperationError === undefined) {
|
||||
fileOperationError = error
|
||||
}
|
||||
}
|
||||
if (fileOperationError !== undefined) throw fileOperationError
|
||||
} catch (error) {
|
||||
operationError = error
|
||||
}
|
||||
try {
|
||||
await directoryHandle.close()
|
||||
} catch (error) {
|
||||
// Preserve an earlier operation error so a cleanup failure cannot hide an
|
||||
// uncertain commit and cause the caller to replay the same batch.
|
||||
if (!committed && operationError === undefined) operationError = error
|
||||
}
|
||||
if (operationError !== undefined) throw operationError
|
||||
},
|
||||
|
||||
copyFileSync(src, dest) {
|
||||
using _ = slowLogging`fs.copyFileSync(${src} → ${dest})`
|
||||
fs.copyFileSync(src, dest)
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { expect, test } from 'bun:test'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { resolve } from 'node:path'
|
||||
|
||||
const testLinuxTraceFile = process.platform === 'linux' ? test : test.skip
|
||||
|
||||
function runFixture(extraEnv: NodeJS.ProcessEnv = {}): {
|
||||
aborted: boolean
|
||||
exitCalled: boolean
|
||||
writeSettled: boolean
|
||||
exitObservedSettledWrite: boolean
|
||||
} {
|
||||
const fixture = resolve(
|
||||
import.meta.dirname,
|
||||
'../test/fixtures/gracefulShutdownTrace.fixture.ts',
|
||||
)
|
||||
const result = spawnSync(process.execPath, [fixture], {
|
||||
encoding: 'utf8',
|
||||
timeout: 15_000,
|
||||
env: { ...process.env, FORCE_COLOR: '0', ...extraEnv },
|
||||
})
|
||||
if (result.error) throw result.error
|
||||
if (result.status !== 0) {
|
||||
throw new Error(
|
||||
`fixture exited with ${result.status}\nstderr:\n${result.stderr}`,
|
||||
)
|
||||
}
|
||||
const resultLine = result.stdout
|
||||
.split('\n')
|
||||
.find(line => line.startsWith('TRACE_SHUTDOWN_RESULT '))
|
||||
expect(resultLine).toBeDefined()
|
||||
return JSON.parse(resultLine!.slice('TRACE_SHUTDOWN_RESULT '.length))
|
||||
}
|
||||
|
||||
testLinuxTraceFile('graceful shutdown drains the interruption trace before process exit', () => {
|
||||
expect(runFixture()).toEqual({
|
||||
aborted: true,
|
||||
exitCalled: true,
|
||||
writeSettled: true,
|
||||
exitObservedSettledWrite: true,
|
||||
})
|
||||
}, 20_000)
|
||||
|
||||
testLinuxTraceFile('graceful shutdown queues otherwise-pending trace records', () => {
|
||||
expect(runFixture({ TRACE_SHUTDOWN_PENDING_ONLY: '1' })).toEqual({
|
||||
aborted: true,
|
||||
exitCalled: true,
|
||||
writeSettled: true,
|
||||
exitObservedSettledWrite: true,
|
||||
})
|
||||
}, 20_000)
|
||||
|
||||
testLinuxTraceFile('graceful shutdown bounds a blocked interruption trace drain', () => {
|
||||
expect(runFixture({ TRACE_SHUTDOWN_BLOCK_WRITE: '1' })).toEqual({
|
||||
aborted: true,
|
||||
exitCalled: true,
|
||||
writeSettled: false,
|
||||
exitObservedSettledWrite: false,
|
||||
})
|
||||
}, 20_000)
|
||||
@@ -39,6 +39,10 @@ import { runCleanupFunctions } from './cleanupRegistry.js'
|
||||
import { createCombinedAbortSignal } from './combinedAbortSignal.js'
|
||||
import { logForDebugging } from './debug.js'
|
||||
import { logForDiagnosticsNoPII } from './diagLogs.js'
|
||||
import {
|
||||
flushInterruptionTrace,
|
||||
waitForInterruptionTraceFlush,
|
||||
} from './interruptionTrace.js'
|
||||
import { isEnvTruthy } from './envUtils.js'
|
||||
import { getCurrentSessionTitle, sessionIdExists } from './sessionStorage.js'
|
||||
import { sleep } from './sleep.js'
|
||||
@@ -534,6 +538,12 @@ export async function gracefulShutdown(
|
||||
}
|
||||
}
|
||||
|
||||
// Root interruption traces are queued off the cancellation path so they
|
||||
// cannot delay abort. Drain that queue before the final process exit; the
|
||||
// drain has its own budget so a blocked target cannot stall normal exit.
|
||||
flushInterruptionTrace('graceful_shutdown')
|
||||
await Promise.race([waitForInterruptionTraceFlush(), sleep(500)])
|
||||
|
||||
forceExit(exitCode)
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,12 @@ import * as realAnalyticsModule from 'src/services/analytics/index.js'
|
||||
import { getCommandQueue, resetCommandQueue } from './messageQueueManager.js'
|
||||
import { createUserMessage } from './messages.js'
|
||||
import * as realProcessUserInputModule from './processUserInput/processUserInput.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
} from './interruptionTrace.js'
|
||||
import type { HandlePromptSubmitParams } from './handlePromptSubmit.js'
|
||||
|
||||
const realAnalytics = { ...realAnalyticsModule }
|
||||
const realProcessUserInput = { ...realProcessUserInputModule }
|
||||
@@ -666,4 +672,61 @@ describe('handlePromptSubmit', () => {
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('traces the explicit submit-interrupt path at runtime', async () => {
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const controller = new AbortController()
|
||||
const { handlePromptSubmit } = await import('./handlePromptSubmit.js')
|
||||
try {
|
||||
const params: HandlePromptSubmitParams = {
|
||||
input: 'echo ready',
|
||||
mode: 'bash',
|
||||
pastedContents: {},
|
||||
helpers: {
|
||||
setCursorOffset: () => {},
|
||||
clearBuffer: () => {},
|
||||
resetHistory: () => {},
|
||||
},
|
||||
onInputChange: () => {},
|
||||
setPastedContents: () => {},
|
||||
abortController: controller,
|
||||
hasInterruptibleToolInProgress: true,
|
||||
streamMode: 'requesting',
|
||||
queryGuard: { isActive: true } as never,
|
||||
isExternalLoading: false,
|
||||
commands: [],
|
||||
messages: [],
|
||||
mainLoopModel: 'sonnet',
|
||||
ideSelection: undefined,
|
||||
querySource: 'repl' as never,
|
||||
setToolJSX: () => {},
|
||||
getToolUseContext: () => ({}) as never,
|
||||
setUserInputOnProcessing: () => {},
|
||||
setAbortController: () => {},
|
||||
onQuery: async () => {},
|
||||
setAppState: () => ({}) as never,
|
||||
}
|
||||
await handlePromptSubmit(params)
|
||||
|
||||
expect(controller.signal.aborted).toBe(true)
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const inputEvent = trace.find(
|
||||
entry => entry.event === 'input.submit_interrupt',
|
||||
)
|
||||
const abortEvent = trace.find(entry => entry.event === 'abort.requested')
|
||||
expect(abortEvent?.source).toBe('interrupt_on_submit')
|
||||
expect(inputEvent).toBeDefined()
|
||||
expect(abortEvent).toBeDefined()
|
||||
expect(typeof inputEvent!.eventId).toBe('string')
|
||||
expect(typeof abortEvent!.causalEventId).toBe('string')
|
||||
expect(abortEvent!.causalEventId).toBe(inputEvent!.eventId)
|
||||
} finally {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -24,6 +24,7 @@ import type { EffortValue } from './effort.js'
|
||||
import type { FileHistoryState } from './fileHistory.js'
|
||||
import { fileHistoryEnabled, fileHistoryMakeSnapshot } from './fileHistory.js'
|
||||
import { gracefulShutdownSync } from './gracefulShutdown.js'
|
||||
import { requestAbort, traceInterruptionEvent } from './interruptionTrace.js'
|
||||
import { enqueue } from './messageQueueManager.js'
|
||||
import { resolveSkillModelOverride } from './model/model.js'
|
||||
import type { ProcessUserInputContext } from './processUserInput/processUserInput.js'
|
||||
@@ -376,7 +377,18 @@ export async function handlePromptSubmit(
|
||||
streamMode:
|
||||
params.streamMode as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
||||
})
|
||||
params.abortController?.abort('interrupt')
|
||||
if (params.abortController) {
|
||||
const causalEventId = traceInterruptionEvent('input.submit_interrupt', {
|
||||
source: 'interrupt_on_submit',
|
||||
subsystem: 'prompt_submit',
|
||||
})
|
||||
requestAbort(params.abortController, 'interrupt', {
|
||||
source: 'interrupt_on_submit',
|
||||
subsystem: 'prompt_submit',
|
||||
controllerRole: 'query-root',
|
||||
causalEventId,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Enqueue with string value + raw pastedContents. Images will be resized
|
||||
|
||||
@@ -0,0 +1,923 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import * as interruptionTraceModule from './interruptionTrace.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__INTERRUPTION_TRACE_CAPACITY_FOR_TESTS,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
flushInterruptionTrace,
|
||||
getInterruptionSignalAbortEventId,
|
||||
registerInterruptionController,
|
||||
requestAbort,
|
||||
traceInterruptionEvent,
|
||||
} from './interruptionTrace.js'
|
||||
import { createCombinedAbortSignal } from './combinedAbortSignal.js'
|
||||
import { createChildAbortController } from './abortController.js'
|
||||
import { logForDiagnosticsNoPII } from './diagLogs.js'
|
||||
import {
|
||||
getFsImplementation,
|
||||
setFsImplementation,
|
||||
type FsOperations,
|
||||
} from './fsOperations.js'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../test/sharedMutationLock.js'
|
||||
|
||||
const originalEnabled = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
const originalFile = process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE
|
||||
const originalDiagnosticsFile = process.env.CLAUDE_CODE_DIAGNOSTICS_FILE
|
||||
let tempDirectory: string | undefined
|
||||
let originalFs: FsOperations
|
||||
const testLinuxTraceFile = process.platform === 'linux' ? test : test.skip
|
||||
const testPosixSymlink = process.platform === 'win32' ? test.skip : test
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock('utils/interruptionTrace.test.ts')
|
||||
originalFs = getFsImplementation()
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE
|
||||
delete process.env.CLAUDE_CODE_DIAGNOSTICS_FILE
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
setFsImplementation(originalFs)
|
||||
if (originalEnabled === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalEnabled
|
||||
if (originalFile === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = originalFile
|
||||
if (originalDiagnosticsFile === undefined) {
|
||||
delete process.env.CLAUDE_CODE_DIAGNOSTICS_FILE
|
||||
} else {
|
||||
process.env.CLAUDE_CODE_DIAGNOSTICS_FILE = originalDiagnosticsFile
|
||||
}
|
||||
if (tempDirectory) {
|
||||
await rm(tempDirectory, { recursive: true, force: true })
|
||||
tempDirectory = undefined
|
||||
}
|
||||
releaseSharedMutationLock()
|
||||
})
|
||||
|
||||
describe('interruptionTrace', () => {
|
||||
test('is a true no-op while disabled and preserves native abort behavior', () => {
|
||||
const controller = new AbortController()
|
||||
registerInterruptionController(controller, { controllerRole: 'root' })
|
||||
traceInterruptionEvent('query.started', { queryId: 'query-1' })
|
||||
requestAbort(controller, 'query-timeout', {
|
||||
source: 'query_guard',
|
||||
controllerRole: 'root',
|
||||
})
|
||||
|
||||
expect(controller.signal.reason).toBe('query-timeout')
|
||||
expect(__getInterruptionTraceSnapshotForTests()).toEqual([])
|
||||
})
|
||||
|
||||
test('correlates controllers and records first-wins plus repeated requests', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const controller = new AbortController()
|
||||
const controllerId = registerInterruptionController(controller, {
|
||||
controllerRole: 'query-root',
|
||||
queryId: 'query-1',
|
||||
})
|
||||
|
||||
requestAbort(controller, 'query-timeout', {
|
||||
source: 'query_guard',
|
||||
queryId: 'query-1',
|
||||
})
|
||||
requestAbort(controller, 'user-cancel', {
|
||||
source: 'cancel_keybinding',
|
||||
queryId: 'query-1',
|
||||
})
|
||||
|
||||
const entries = __getInterruptionTraceSnapshotForTests()
|
||||
const requested = entries.find(entry => entry.event === 'abort.requested')
|
||||
const observed = entries.find(entry => entry.event === 'signal.observed')
|
||||
const repeated = entries.find(entry => entry.event === 'abort.repeated')
|
||||
expect(controller.signal.reason).toBe('query-timeout')
|
||||
expect(requested?.controllerId).toBe(controllerId)
|
||||
expect(requested?.normalizedReason).toBe('query-timeout')
|
||||
expect(requested?.abortStackFingerprint).toMatch(/^[a-f0-9]{16}$/)
|
||||
expect(requested).not.toHaveProperty('abortCallSites')
|
||||
expect(typeof requested?.eventId).toBe('string')
|
||||
expect(typeof observed?.firstAbortEventId).toBe('string')
|
||||
expect(typeof repeated?.firstAbortEventId).toBe('string')
|
||||
expect(observed!.firstAbortEventId).toBe(requested!.eventId)
|
||||
expect(repeated!.firstAbortEventId).toBe(requested!.eventId)
|
||||
expect(repeated?.existingNormalizedReason).toBe('query-timeout')
|
||||
expect(repeated?.attemptedNormalizedReason).toBe('user-abort')
|
||||
expect(repeated?.outcome).toBe('ignored_first_abort_wins')
|
||||
expect(repeated?.repeatedCount).toBe(1)
|
||||
})
|
||||
|
||||
test('links a combined signal abort to the winning parent request', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const parent = new AbortController()
|
||||
registerInterruptionController(parent, {
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
const combined = createCombinedAbortSignal(parent.signal, {
|
||||
trace: {
|
||||
subsystem: 'trace-test',
|
||||
controllerRole: 'combined',
|
||||
},
|
||||
})
|
||||
|
||||
requestAbort(parent, 'query-timeout', {
|
||||
source: 'query_guard',
|
||||
subsystem: 'trace-test',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
|
||||
const requested = __getInterruptionTraceSnapshotForTests().filter(
|
||||
entry => entry.event === 'abort.requested',
|
||||
)
|
||||
expect(combined.signal.reason).toBe('query-timeout')
|
||||
expect(requested).toHaveLength(2)
|
||||
expect(requested[1]?.causalEventId).toBe(requested[0]?.eventId)
|
||||
expect(requested[1]?.winningParentControllerId).toBe(
|
||||
requested[0]?.controllerId,
|
||||
)
|
||||
combined.cleanup()
|
||||
})
|
||||
|
||||
test('assigns a causal event id when a registered signal aborts natively', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const controller = new AbortController()
|
||||
registerInterruptionController(controller, { controllerRole: 'external' })
|
||||
|
||||
controller.abort('external-abort')
|
||||
|
||||
const observed = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'signal.observed',
|
||||
)
|
||||
expect(observed).toBeDefined()
|
||||
expect(typeof observed?.eventId).toBe('string')
|
||||
expect(getInterruptionSignalAbortEventId(controller.signal)).toBe(
|
||||
observed!.eventId,
|
||||
)
|
||||
|
||||
requestAbort(controller, 'second-abort', {
|
||||
source: 'native-abort-test',
|
||||
controllerRole: 'external',
|
||||
})
|
||||
const repeated = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'abort.repeated',
|
||||
)
|
||||
expect(repeated).toBeDefined()
|
||||
expect(typeof repeated?.firstAbortEventId).toBe('string')
|
||||
expect(repeated!.firstAbortEventId).toBe(observed!.eventId)
|
||||
})
|
||||
|
||||
test('links a native AbortSignal.any result to its winning parent', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const caller = new AbortController()
|
||||
const deadline = new AbortController()
|
||||
const combined = AbortSignal.any([caller.signal, deadline.signal])
|
||||
interruptionTraceModule.traceCombinedAbortSignal(
|
||||
combined,
|
||||
[caller.signal, deadline.signal], {
|
||||
subsystem: 'native-any-test',
|
||||
controllerRole: 'request-combined',
|
||||
},
|
||||
)
|
||||
|
||||
caller.abort('user-cancel')
|
||||
|
||||
const entries = __getInterruptionTraceSnapshotForTests()
|
||||
const parentObserved = entries.find(
|
||||
entry =>
|
||||
entry.event === 'signal.observed' &&
|
||||
entry.controllerRole === 'combined-parent',
|
||||
)
|
||||
const combinedObserved = entries.find(
|
||||
entry =>
|
||||
entry.event === 'signal.observed' &&
|
||||
entry.controllerRole === 'request-combined',
|
||||
)
|
||||
expect(parentObserved).toBeDefined()
|
||||
expect(combinedObserved).toMatchObject({
|
||||
subsystem: 'native-any-test',
|
||||
causalEventId: parentObserved!.eventId,
|
||||
winningParentControllerId: parentObserved!.controllerId,
|
||||
})
|
||||
})
|
||||
|
||||
test('records permission abort resolution with the input causal edge', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const inputEventId = traceInterruptionEvent('input.ctrl_c')
|
||||
|
||||
interruptionTraceModule.tracePermissionAbortResolution(
|
||||
'ctrl_c',
|
||||
inputEventId,
|
||||
'remote-permission',
|
||||
)
|
||||
|
||||
expect(__getInterruptionTraceSnapshotForTests().at(-1)).toMatchObject({
|
||||
event: 'permission.abort_resolved',
|
||||
source: 'ctrl_c',
|
||||
subsystem: 'remote-permission',
|
||||
causalEventId: inputEventId,
|
||||
outcome: 'denied',
|
||||
})
|
||||
})
|
||||
|
||||
testLinuxTraceFile('observes and flushes a query-root controller already aborted when registered', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl'
|
||||
let writes = 0
|
||||
setFsImplementation({
|
||||
...originalFs,
|
||||
appendRegularFile: async () => {
|
||||
writes++
|
||||
},
|
||||
})
|
||||
const controller = new AbortController()
|
||||
controller.abort('external-abort')
|
||||
|
||||
registerInterruptionController(controller, { controllerRole: 'query-root' })
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
|
||||
const observed = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'signal.observed',
|
||||
)
|
||||
expect(observed?.controllerRole).toBe('query-root')
|
||||
expect(getInterruptionSignalAbortEventId(controller.signal)).toBe(
|
||||
observed?.eventId,
|
||||
)
|
||||
expect(writes).toBe(1)
|
||||
})
|
||||
|
||||
testLinuxTraceFile('preserves an established query-root role when creating a child', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl'
|
||||
let writes = 0
|
||||
setFsImplementation({
|
||||
...originalFs,
|
||||
appendRegularFile: async () => {
|
||||
writes++
|
||||
},
|
||||
})
|
||||
const parent = new AbortController()
|
||||
registerInterruptionController(parent, {
|
||||
controllerRole: 'query-root',
|
||||
queryId: 'query-root-1',
|
||||
queryGeneration: 3,
|
||||
})
|
||||
createChildAbortController(parent)
|
||||
|
||||
requestAbort(parent, 'user-cancel', {
|
||||
source: 'cancel_keybinding',
|
||||
controllerRole: 'tool',
|
||||
})
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
|
||||
const observed = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'signal.observed' &&
|
||||
entry.normalizedReason === 'user-abort',
|
||||
)
|
||||
const requested = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'abort.requested',
|
||||
)
|
||||
expect(requested).toMatchObject({
|
||||
source: 'cancel_keybinding',
|
||||
controllerRole: 'query-root',
|
||||
queryId: 'query-root-1',
|
||||
queryGeneration: 3,
|
||||
})
|
||||
expect(observed?.controllerRole).toBe('query-root')
|
||||
expect(writes).toBe(1)
|
||||
})
|
||||
|
||||
test('replaces provisional parent and child roles with concrete lifecycle roles', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const parent = new AbortController()
|
||||
const child = createChildAbortController(parent)
|
||||
|
||||
requestAbort(child, 'sibling_error', {
|
||||
source: 'sibling_error',
|
||||
subsystem: 'streaming_tool_executor',
|
||||
controllerRole: 'sibling-tools',
|
||||
})
|
||||
requestAbort(parent, 'task_stop', {
|
||||
source: 'task_stop',
|
||||
subsystem: 'local_agent_task',
|
||||
controllerRole: 'background-agent',
|
||||
subagentId: 'agent-1',
|
||||
})
|
||||
|
||||
const requested = __getInterruptionTraceSnapshotForTests().filter(
|
||||
entry => entry.event === 'abort.requested',
|
||||
)
|
||||
expect(requested).toHaveLength(2)
|
||||
expect(requested[0]).toMatchObject({
|
||||
source: 'sibling_error',
|
||||
controllerRole: 'sibling-tools',
|
||||
})
|
||||
expect(requested[1]).toMatchObject({
|
||||
source: 'task_stop',
|
||||
controllerRole: 'background-agent',
|
||||
subagentId: 'agent-1',
|
||||
})
|
||||
const observed = __getInterruptionTraceSnapshotForTests().filter(
|
||||
entry => entry.event === 'signal.observed',
|
||||
)
|
||||
expect(observed).toHaveLength(2)
|
||||
expect(observed[0]?.controllerRole).toBe('sibling-tools')
|
||||
expect(observed[1]).toMatchObject({
|
||||
controllerRole: 'background-agent',
|
||||
subagentId: 'agent-1',
|
||||
})
|
||||
})
|
||||
|
||||
test('retains a child owner when the parent aborts before a later request', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const parent = new AbortController()
|
||||
const child = createChildAbortController(parent, undefined, {
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: 'agent-1',
|
||||
})
|
||||
|
||||
requestAbort(parent, undefined, {
|
||||
source: 'cancel_keybinding',
|
||||
subsystem: 'query_engine',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
|
||||
const childObserved = __getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'signal.observed' &&
|
||||
entry.controllerRole === 'subagent-lifecycle',
|
||||
)
|
||||
expect(child.signal.aborted).toBe(true)
|
||||
expect(childObserved).toMatchObject({
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: 'agent-1',
|
||||
})
|
||||
})
|
||||
|
||||
test('throwing abort-reason accessors cannot block native or combined aborts', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const reason = new Proxy(
|
||||
{},
|
||||
{
|
||||
get() {
|
||||
throw new Error('reason getter must stay isolated')
|
||||
},
|
||||
},
|
||||
)
|
||||
const direct = new AbortController()
|
||||
expect(() =>
|
||||
requestAbort(direct, reason, {
|
||||
source: 'throwing-reason-test',
|
||||
controllerRole: 'query-root',
|
||||
}),
|
||||
).not.toThrow()
|
||||
expect(direct.signal.aborted).toBe(true)
|
||||
expect(direct.signal.reason).toBe(reason)
|
||||
|
||||
const parent = new AbortController()
|
||||
const combined = createCombinedAbortSignal(parent.signal)
|
||||
expect(() => parent.abort(reason)).not.toThrow()
|
||||
expect(combined.signal.aborted).toBe(true)
|
||||
expect(combined.signal.reason).toBe(reason)
|
||||
combined.cleanup()
|
||||
})
|
||||
|
||||
test('does not persist arbitrary custom error names', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const reason = new Error('private reason message')
|
||||
reason.name = 'private customer prompt'
|
||||
|
||||
traceInterruptionEvent('custom.error', { reason, error: reason })
|
||||
|
||||
const serialized = JSON.stringify(__getInterruptionTraceSnapshotForTests())
|
||||
expect(serialized).not.toContain('private customer prompt')
|
||||
expect(serialized).not.toContain('private reason message')
|
||||
const entry = __getInterruptionTraceSnapshotForTests().at(-1)
|
||||
expect(entry?.rawReasonType).toBe('Error:Error')
|
||||
expect(entry?.safeErrorIdentity).toBe('Error')
|
||||
})
|
||||
|
||||
test('preserves only standardized DOMException names', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const standard = new DOMException('private message', 'AbortError')
|
||||
const custom = new DOMException('private message', 'private customer prompt')
|
||||
|
||||
traceInterruptionEvent('standard.dom.error', {
|
||||
reason: standard,
|
||||
error: standard,
|
||||
})
|
||||
traceInterruptionEvent('custom.dom.error', {
|
||||
reason: custom,
|
||||
error: custom,
|
||||
})
|
||||
|
||||
const entries = __getInterruptionTraceSnapshotForTests()
|
||||
expect(entries.at(-2)?.rawReasonType).toBe('DOMException:AbortError')
|
||||
expect(entries.at(-2)?.safeErrorIdentity).toBe('DOMException:AbortError')
|
||||
expect(entries.at(-1)?.rawReasonType).toBe('DOMException')
|
||||
expect(entries.at(-1)?.safeErrorIdentity).toBe('DOMException')
|
||||
expect(JSON.stringify(entries)).not.toContain('private customer prompt')
|
||||
expect(JSON.stringify(entries)).not.toContain('private message')
|
||||
})
|
||||
|
||||
test('keeps only the newest allowlisted records up to capacity', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = 'true'
|
||||
const emitted = __INTERRUPTION_TRACE_CAPACITY_FOR_TESTS + 88
|
||||
for (let index = 0; index < emitted; index++) {
|
||||
traceInterruptionEvent('stream.progress', {
|
||||
rawByteCount: index,
|
||||
// Runtime extras are deliberately ignored by the allowlist boundary.
|
||||
...({ prompt: 'must-not-appear' } as Record<string, unknown>),
|
||||
})
|
||||
}
|
||||
|
||||
const entries = __getInterruptionTraceSnapshotForTests()
|
||||
expect(entries).toHaveLength(__INTERRUPTION_TRACE_CAPACITY_FOR_TESTS)
|
||||
expect(entries[0]?.sequence).toBe(emitted - __INTERRUPTION_TRACE_CAPACITY_FOR_TESTS + 1)
|
||||
expect(entries.at(-1)?.sequence).toBe(emitted)
|
||||
expect(JSON.stringify(entries)).not.toContain('must-not-appear')
|
||||
expect(JSON.stringify(entries)).not.toContain('prompt')
|
||||
})
|
||||
|
||||
testLinuxTraceFile('flushes valid JSONL once to an explicit absolute path', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-interrupt-trace-'))
|
||||
const traceFile = join(tempDirectory, 'trace.jsonl')
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = traceFile
|
||||
|
||||
traceInterruptionEvent('query.started', {
|
||||
queryId: 'query-1',
|
||||
model: 'gpt-test',
|
||||
})
|
||||
flushInterruptionTrace('test')
|
||||
flushInterruptionTrace('test-repeat')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
|
||||
const lines = (await readFile(traceFile, 'utf8')).trim().split('\n')
|
||||
const entries = lines.map(line => JSON.parse(line) as { event: string })
|
||||
expect(entries.map(entry => entry.event)).toEqual([
|
||||
'query.started',
|
||||
'trace.flush',
|
||||
])
|
||||
})
|
||||
|
||||
testLinuxTraceFile('flushes every pending record when the ring is at capacity', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-interrupt-trace-'))
|
||||
const traceFile = join(tempDirectory, 'trace.jsonl')
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = traceFile
|
||||
for (let index = 0; index < __INTERRUPTION_TRACE_CAPACITY_FOR_TESTS; index++) {
|
||||
traceInterruptionEvent('stream.progress', { rawByteCount: index })
|
||||
}
|
||||
|
||||
flushInterruptionTrace('capacity')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
|
||||
const entries = (await readFile(traceFile, 'utf8'))
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(line => JSON.parse(line) as { sequence: number; event: string })
|
||||
expect(entries).toHaveLength(__INTERRUPTION_TRACE_CAPACITY_FOR_TESTS + 1)
|
||||
expect(entries[0]?.sequence).toBe(1)
|
||||
expect(entries.at(-1)?.event).toBe('trace.flush')
|
||||
})
|
||||
|
||||
testLinuxTraceFile('retains pending records after a failed write and retries them', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl'
|
||||
let failWrites = true
|
||||
let writeAttempts = 0
|
||||
const successfulWrites: string[] = []
|
||||
setFsImplementation({
|
||||
...originalFs,
|
||||
mkdirSync: () => {},
|
||||
appendRegularFile: async (_path, data) => {
|
||||
writeAttempts++
|
||||
if (failWrites) throw new Error('synthetic write failure')
|
||||
successfulWrites.push(data)
|
||||
},
|
||||
})
|
||||
|
||||
traceInterruptionEvent('first')
|
||||
flushInterruptionTrace('failed')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
expect(writeAttempts).toBe(1)
|
||||
failWrites = false
|
||||
traceInterruptionEvent('second')
|
||||
flushInterruptionTrace('retry')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
|
||||
expect(writeAttempts).toBe(3)
|
||||
expect(successfulWrites).toHaveLength(2)
|
||||
const events = successfulWrites.flatMap(write =>
|
||||
write
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(line => (JSON.parse(line) as { event: string }).event),
|
||||
)
|
||||
expect(events).toEqual(['first', 'trace.flush', 'second', 'trace.flush'])
|
||||
})
|
||||
|
||||
testLinuxTraceFile('captures the enabled output target before a detached flush starts', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/captured/trace.jsonl'
|
||||
const writes: Array<{ path: string; data: string }> = []
|
||||
setFsImplementation({
|
||||
...originalFs,
|
||||
appendRegularFile: async (path, data) => {
|
||||
writes.push({ path, data })
|
||||
},
|
||||
})
|
||||
|
||||
traceInterruptionEvent('before_restore')
|
||||
flushInterruptionTrace('captured-target')
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
|
||||
expect(writes).toHaveLength(1)
|
||||
expect(writes[0]?.path).toBe('/captured/trace.jsonl')
|
||||
expect(writes[0]?.data).toContain('"event":"before_restore"')
|
||||
expect(writes[0]?.data).toContain('"event":"trace.flush"')
|
||||
})
|
||||
|
||||
testLinuxTraceFile('keeps a retry batch bound to its original output target', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/first/trace.jsonl'
|
||||
const writes: Array<{ path: string; data: string }> = []
|
||||
setFsImplementation({
|
||||
...originalFs,
|
||||
appendRegularFile: async (path, data) => {
|
||||
writes.push({ path, data })
|
||||
if (writes.length === 1) throw new Error('synthetic retryable failure')
|
||||
},
|
||||
})
|
||||
|
||||
traceInterruptionEvent('first_target')
|
||||
flushInterruptionTrace('first')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/second/trace.jsonl'
|
||||
traceInterruptionEvent('second_target')
|
||||
flushInterruptionTrace('second')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
|
||||
expect(writes.map(write => write.path)).toEqual([
|
||||
'/first/trace.jsonl',
|
||||
'/first/trace.jsonl',
|
||||
'/second/trace.jsonl',
|
||||
])
|
||||
expect(writes[1]?.data).toContain('"event":"first_target"')
|
||||
expect(writes[2]?.data).toContain('"event":"second_target"')
|
||||
expect(writes[2]?.data).not.toContain('"event":"first_target"')
|
||||
})
|
||||
|
||||
testLinuxTraceFile('retains an in-flight failed batch outside the bounded ring', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl'
|
||||
let rejectFirstWrite!: (error: Error) => void
|
||||
let markFirstWriteStarted!: () => void
|
||||
const firstWriteStarted = new Promise<void>(resolve => {
|
||||
markFirstWriteStarted = resolve
|
||||
})
|
||||
const firstWriteBlocked = new Promise<void>((_resolve, reject) => {
|
||||
rejectFirstWrite = reject
|
||||
})
|
||||
const writes: string[] = []
|
||||
setFsImplementation({
|
||||
...originalFs,
|
||||
appendRegularFile: async (_path, data) => {
|
||||
if (writes.length === 0) {
|
||||
writes.push('failed')
|
||||
markFirstWriteStarted()
|
||||
await firstWriteBlocked
|
||||
return
|
||||
}
|
||||
writes.push(data)
|
||||
},
|
||||
})
|
||||
|
||||
traceInterruptionEvent('must_survive')
|
||||
flushInterruptionTrace('blocked')
|
||||
await firstWriteStarted
|
||||
for (let index = 0; index < __INTERRUPTION_TRACE_CAPACITY_FOR_TESTS + 32; index++) {
|
||||
traceInterruptionEvent('newer', { rawByteCount: index })
|
||||
}
|
||||
rejectFirstWrite(new Error('synthetic write failure'))
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
flushInterruptionTrace('retry')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
|
||||
expect(writes).toHaveLength(3)
|
||||
expect(writes[1]).toContain('"event":"must_survive"')
|
||||
})
|
||||
|
||||
testLinuxTraceFile('does not replay a batch after an uncertain append failure', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl'
|
||||
const writes: string[] = []
|
||||
setFsImplementation({
|
||||
...originalFs,
|
||||
appendRegularFile: async (_path, data) => {
|
||||
writes.push(data)
|
||||
if (writes.length === 1) {
|
||||
throw Object.assign(new Error('partial append'), {
|
||||
code: 'ERR_DIAGNOSTIC_APPEND_UNCERTAIN',
|
||||
})
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
traceInterruptionEvent('first')
|
||||
flushInterruptionTrace('uncertain')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
traceInterruptionEvent('second')
|
||||
flushInterruptionTrace('later')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
|
||||
expect(writes).toHaveLength(2)
|
||||
expect(writes[0]).toContain('"event":"first"')
|
||||
expect(writes[1]).not.toContain('"event":"first"')
|
||||
expect(writes[1]).toContain('"event":"second"')
|
||||
})
|
||||
|
||||
testLinuxTraceFile('rejects an existing non-regular trace target', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-interrupt-trace-'))
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = tempDirectory
|
||||
traceInterruptionEvent('pending')
|
||||
|
||||
flushInterruptionTrace('non-regular')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().some(
|
||||
entry => entry.event === 'trace.flush',
|
||||
),
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
testLinuxTraceFile('rejects symlink targets and creates private files and directories', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-interrupt-trace-'))
|
||||
const target = join(tempDirectory, 'target.jsonl')
|
||||
const link = join(tempDirectory, 'trace-link.jsonl')
|
||||
await writeFile(target, '')
|
||||
await symlink(target, link)
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = link
|
||||
traceInterruptionEvent('symlink-pending')
|
||||
flushInterruptionTrace('symlink')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
expect(await readFile(target, 'utf8')).toBe('')
|
||||
|
||||
// The rejected batch remains bound to its original target for retry.
|
||||
// Reset before exercising independent creation and permission behavior.
|
||||
__resetInterruptionTraceForTests()
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
|
||||
const privateDirectory = join(tempDirectory, 'private')
|
||||
const privateTrace = join(privateDirectory, 'trace.jsonl')
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = privateTrace
|
||||
traceInterruptionEvent('private-target-pending')
|
||||
flushInterruptionTrace('private-target')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
expect((await stat(privateDirectory)).mode & 0o777).toBe(0o700)
|
||||
expect((await stat(privateTrace)).mode & 0o777).toBe(0o600)
|
||||
|
||||
const existingTrace = join(tempDirectory, 'existing.jsonl')
|
||||
await writeFile(existingTrace, '', { mode: 0o644 })
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = existingTrace
|
||||
traceInterruptionEvent('existing-private-target')
|
||||
flushInterruptionTrace('existing-private-target')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
expect((await stat(existingTrace)).mode & 0o777).toBe(0o600)
|
||||
})
|
||||
|
||||
testLinuxTraceFile('rejects a trace target beneath a symlinked parent directory', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-interrupt-trace-'))
|
||||
const realDirectory = join(tempDirectory, 'real-parent')
|
||||
const linkedDirectory = join(tempDirectory, 'linked-parent')
|
||||
const realTrace = join(realDirectory, 'trace.jsonl')
|
||||
await mkdir(realDirectory)
|
||||
await symlink(realDirectory, linkedDirectory, 'dir')
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = join(
|
||||
linkedDirectory,
|
||||
'trace.jsonl',
|
||||
)
|
||||
|
||||
traceInterruptionEvent('symlinked-parent-pending')
|
||||
flushInterruptionTrace('symlinked-parent')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
|
||||
const result = await readFile(realTrace, 'utf8').catch(
|
||||
error => (error as NodeJS.ErrnoException).code,
|
||||
)
|
||||
expect(result).toBe('ENOENT')
|
||||
})
|
||||
|
||||
testPosixSymlink('preserves legacy diagnostics append-through-symlink behavior', async () => {
|
||||
tempDirectory = await mkdtemp(join(tmpdir(), 'openclaude-diagnostics-'))
|
||||
const target = join(tempDirectory, 'target.jsonl')
|
||||
const link = join(tempDirectory, 'diagnostics-link.jsonl')
|
||||
await writeFile(target, '')
|
||||
await symlink(target, link)
|
||||
process.env.CLAUDE_CODE_DIAGNOSTICS_FILE = link
|
||||
|
||||
logForDiagnosticsNoPII('info', 'symlink-test')
|
||||
|
||||
expect(await readFile(target, 'utf8')).toContain('symlink-test')
|
||||
delete process.env.CLAUDE_CODE_DIAGNOSTICS_FILE
|
||||
})
|
||||
|
||||
test('does not block native abort while a trace append is pending', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl'
|
||||
let releaseWrite!: () => void
|
||||
const writeBlocked = new Promise<void>(resolve => {
|
||||
releaseWrite = resolve
|
||||
})
|
||||
setFsImplementation({
|
||||
...originalFs,
|
||||
appendRegularFile: async () => writeBlocked,
|
||||
})
|
||||
const controller = new AbortController()
|
||||
|
||||
try {
|
||||
requestAbort(controller, 'user-cancel', {
|
||||
source: 'cancel_keybinding',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
|
||||
expect(controller.signal.aborted).toBe(true)
|
||||
} finally {
|
||||
releaseWrite()
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
}
|
||||
})
|
||||
|
||||
testLinuxTraceFile('keeps sequence IDs unique and drains later events during an async flush', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl'
|
||||
let releaseFirstWrite!: () => void
|
||||
let markFirstWriteStarted!: () => void
|
||||
const firstWriteStarted = new Promise<void>(resolve => {
|
||||
markFirstWriteStarted = resolve
|
||||
})
|
||||
const firstWriteBlocked = new Promise<void>(resolve => {
|
||||
releaseFirstWrite = resolve
|
||||
})
|
||||
const writes: string[] = []
|
||||
setFsImplementation({
|
||||
...originalFs,
|
||||
appendRegularFile: async (_path, data) => {
|
||||
writes.push(data)
|
||||
if (writes.length === 1) {
|
||||
markFirstWriteStarted()
|
||||
await firstWriteBlocked
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
traceInterruptionEvent('before')
|
||||
flushInterruptionTrace('first')
|
||||
try {
|
||||
await firstWriteStarted
|
||||
traceInterruptionEvent('during_one')
|
||||
traceInterruptionEvent('during_two')
|
||||
} finally {
|
||||
releaseFirstWrite()
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
}
|
||||
const persistedEvents = writes.flatMap(write =>
|
||||
write
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(line => (JSON.parse(line) as { event: string }).event),
|
||||
)
|
||||
expect(persistedEvents.filter(event => event === 'before')).toHaveLength(1)
|
||||
expect(persistedEvents.filter(event => event === 'during_one')).toHaveLength(1)
|
||||
expect(persistedEvents.filter(event => event === 'during_two')).toHaveLength(1)
|
||||
const snapshot = __getInterruptionTraceSnapshotForTests()
|
||||
expect(new Set(snapshot.map(entry => entry.eventId)).size).toBe(snapshot.length)
|
||||
expect(snapshot.map(entry => entry.sequence)).toEqual(
|
||||
[...snapshot.map(entry => entry.sequence)].sort((left, right) => left - right),
|
||||
)
|
||||
})
|
||||
|
||||
test('never persists dynamic function names from abort stacks', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const controller = new AbortController()
|
||||
const customerSpecificHandlerName = () => {
|
||||
requestAbort(controller, 'user-cancel', { source: 'test' })
|
||||
}
|
||||
|
||||
customerSpecificHandlerName()
|
||||
|
||||
const serialized = JSON.stringify(__getInterruptionTraceSnapshotForTests())
|
||||
expect(serialized).not.toContain('customerSpecificHandlerName')
|
||||
expect(serialized).not.toContain('abortCallSites')
|
||||
})
|
||||
|
||||
test('does not write for relative paths and isolates write failures', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
traceInterruptionEvent('query.started')
|
||||
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = 'relative-trace.jsonl'
|
||||
expect(() => flushInterruptionTrace('relative')).not.toThrow()
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/proc/openclaude/trace.jsonl'
|
||||
expect(() => flushInterruptionTrace('unwritable')).not.toThrow()
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
})
|
||||
|
||||
test('discards trace-file batches without retrying on unsupported platforms', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE_FILE = '/trace.jsonl'
|
||||
const writtenData: string[] = []
|
||||
setFsImplementation({
|
||||
...originalFs,
|
||||
appendRegularFile: async (_path, data) => {
|
||||
writtenData.push(data)
|
||||
},
|
||||
})
|
||||
const platformDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')
|
||||
Object.defineProperty(process, 'platform', {
|
||||
value: 'darwin',
|
||||
configurable: true,
|
||||
})
|
||||
try {
|
||||
traceInterruptionEvent('pending')
|
||||
flushInterruptionTrace('posix-platform')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
traceInterruptionEvent('later')
|
||||
flushInterruptionTrace('posix-platform-later')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
expect(writtenData).toEqual([])
|
||||
} finally {
|
||||
if (platformDescriptor) {
|
||||
Object.defineProperty(process, 'platform', platformDescriptor)
|
||||
}
|
||||
}
|
||||
if (process.platform === 'linux') {
|
||||
traceInterruptionEvent('after_restore')
|
||||
flushInterruptionTrace('restored-platform')
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
expect(writtenData).toHaveLength(1)
|
||||
const restoredWrite = writtenData[0] ?? ''
|
||||
expect(restoredWrite).toContain('"event":"after_restore"')
|
||||
expect(restoredWrite).not.toContain('"event":"pending"')
|
||||
expect(restoredWrite).not.toContain('"event":"later"')
|
||||
}
|
||||
})
|
||||
|
||||
test('redacts secret-shaped values and absolute local paths', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
traceInterruptionEvent('provider.failed', {
|
||||
model: 'AKIA1234567890ABCDEF',
|
||||
providerRoute: 'github_pat_1234567890abcdef',
|
||||
attemptId: '\\\\server\\share\\private',
|
||||
queryId: 'prefix(/srv/private/project)',
|
||||
error: new Error('message content is never serialized'),
|
||||
})
|
||||
|
||||
const serialized = JSON.stringify(__getInterruptionTraceSnapshotForTests())
|
||||
expect(serialized).toContain('[redacted]')
|
||||
expect(serialized).toContain('Error')
|
||||
expect(serialized).not.toContain('AKIA1234567890ABCDEF')
|
||||
expect(serialized).not.toContain('github_pat_1234567890abcdef')
|
||||
expect(serialized).not.toContain('server')
|
||||
expect(serialized).not.toContain('/srv/private/project')
|
||||
expect(serialized).not.toContain('message content')
|
||||
})
|
||||
|
||||
test('never serializes secret-shaped or path-shaped abort reasons', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const secretReason = 'github_pat_1234567890abcdef'
|
||||
const pathReason = '/srv/private/project/abort-reason'
|
||||
|
||||
traceInterruptionEvent('abort.requested', { reason: secretReason })
|
||||
traceInterruptionEvent('abort.repeated', {
|
||||
existingReason: pathReason,
|
||||
attemptedReason: secretReason,
|
||||
})
|
||||
|
||||
const snapshot = __getInterruptionTraceSnapshotForTests()
|
||||
const serialized = JSON.stringify(snapshot)
|
||||
expect(snapshot).toHaveLength(2)
|
||||
expect(snapshot[0]?.normalizedReason).toBe('unknown-abort')
|
||||
expect(snapshot[1]?.existingNormalizedReason).toBe('unknown-abort')
|
||||
expect(snapshot[1]?.attemptedNormalizedReason).toBe('unknown-abort')
|
||||
expect(serialized).not.toContain(secretReason)
|
||||
expect(serialized).not.toContain(pathReason)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,790 @@
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { isAbsolute, posix, win32 } from 'node:path'
|
||||
import {
|
||||
monitorEventLoopDelay,
|
||||
type IntervalHistogram,
|
||||
} from 'node:perf_hooks'
|
||||
import { normalizeAbortReason } from './abortReasons.js'
|
||||
import { appendDiagnosticsNoPII } from './diagLogs.js'
|
||||
import { redactHomePath, redactLikelySecrets } from './redaction.js'
|
||||
|
||||
const TRACE_CAPACITY = 512
|
||||
export const __INTERRUPTION_TRACE_CAPACITY_FOR_TESTS = TRACE_CAPACITY
|
||||
const TRACE_SCHEMA_VERSION = 1
|
||||
const TRACE_ENABLED_ENV = 'OPENCLAUDE_INTERRUPT_TRACE'
|
||||
const TRACE_FILE_ENV = 'OPENCLAUDE_INTERRUPT_TRACE_FILE'
|
||||
|
||||
export type InterruptionTraceFields = {
|
||||
source?: string
|
||||
subsystem?: string
|
||||
phase?: string
|
||||
queryId?: string
|
||||
queryGeneration?: number
|
||||
querySource?: string
|
||||
parentQueryId?: string
|
||||
subagentId?: string
|
||||
providerRoute?: string
|
||||
transport?: string
|
||||
model?: string
|
||||
attemptId?: string
|
||||
controllerRole?: string
|
||||
parentControllerIds?: readonly string[]
|
||||
winningParentControllerId?: string
|
||||
causalEventId?: string
|
||||
trigger?: string
|
||||
outcome?: string
|
||||
reason?: unknown
|
||||
existingReason?: unknown
|
||||
attemptedReason?: unknown
|
||||
error?: unknown
|
||||
elapsedQueryMs?: number
|
||||
sinceLastActivityMs?: number
|
||||
sinceLastRawByteMs?: number
|
||||
sinceLastParsedFrameMs?: number
|
||||
sinceLastYieldMs?: number
|
||||
rawByteCount?: number
|
||||
parsedFrameCount?: number
|
||||
controlFrameCount?: number
|
||||
ignoredFrameCount?: number
|
||||
ignoredParsedFrameCount?: number
|
||||
yieldedEventCount?: number
|
||||
activeApiCallCount?: number
|
||||
activeToolUseCount?: number
|
||||
leaseCount?: number
|
||||
suspendCount?: number
|
||||
repeatedCount?: number
|
||||
eventLoopDelayMaxMs?: number
|
||||
eventLoopDelayMeanMs?: number
|
||||
}
|
||||
|
||||
type SafeTraceFields = Omit<
|
||||
InterruptionTraceFields,
|
||||
| 'reason'
|
||||
| 'existingReason'
|
||||
| 'attemptedReason'
|
||||
| 'error'
|
||||
| 'parentControllerIds'
|
||||
> & {
|
||||
normalizedReason?: string
|
||||
rawReasonType?: string
|
||||
existingNormalizedReason?: string
|
||||
existingRawReasonType?: string
|
||||
attemptedNormalizedReason?: string
|
||||
attemptedRawReasonType?: string
|
||||
safeErrorIdentity?: string
|
||||
parentControllerIds?: string[]
|
||||
}
|
||||
|
||||
export type InterruptionTraceEntry = SafeTraceFields & {
|
||||
schemaVersion: number
|
||||
sequence: number
|
||||
eventId: string
|
||||
traceSessionId: string
|
||||
timestamp: string
|
||||
monotonicMs: number
|
||||
clockDeltaMs: number
|
||||
event: string
|
||||
controllerId?: string
|
||||
firstAbortEventId?: string
|
||||
abortStackFingerprint?: string
|
||||
}
|
||||
|
||||
type ControllerTraceState = {
|
||||
id: string
|
||||
firstAbortEventId?: string
|
||||
repeatedCount: number
|
||||
fields: InterruptionTraceFields
|
||||
provisionalRole: boolean
|
||||
}
|
||||
|
||||
type SignalTraceState = {
|
||||
id: string
|
||||
fields: InterruptionTraceFields
|
||||
getAbortFields?: () => InterruptionTraceFields
|
||||
}
|
||||
|
||||
let traceSessionId = ''
|
||||
let sequence = 0
|
||||
let startedWallMs = Date.now()
|
||||
let startedMonotonicMs = performance.now()
|
||||
let ring: InterruptionTraceEntry[] = []
|
||||
let flushedThroughSequence = 0
|
||||
let controllerCounter = 0
|
||||
let signalCounter = 0
|
||||
let controllerStates = new WeakMap<AbortController, ControllerTraceState>()
|
||||
let signalIds = new WeakMap<AbortSignal, string>()
|
||||
let signalAbortEventIds = new WeakMap<AbortSignal, string>()
|
||||
let signalAbortSources = new WeakMap<AbortSignal, string>()
|
||||
let signalStates = new WeakMap<AbortSignal, SignalTraceState>()
|
||||
let errorCausalEventIds = new WeakMap<object, string>()
|
||||
let eventLoopDelay: IntervalHistogram | undefined
|
||||
let flushQueue: Promise<void> = Promise.resolve()
|
||||
let retryBatch:
|
||||
| {
|
||||
entries: readonly InterruptionTraceEntry[]
|
||||
marker: InterruptionTraceEntry
|
||||
logFile: string
|
||||
}
|
||||
| undefined
|
||||
|
||||
function isEnabled(): boolean {
|
||||
const value = process.env[TRACE_ENABLED_ENV]?.toLowerCase()
|
||||
return value === '1' || value === 'true'
|
||||
}
|
||||
|
||||
export function isInterruptionTraceEnabled(): boolean {
|
||||
return isEnabled()
|
||||
}
|
||||
|
||||
function safeString(value: unknown): string | undefined {
|
||||
if (typeof value !== 'string' || value.length === 0) return undefined
|
||||
const clipped = value.slice(0, 160)
|
||||
const redacted = redactLikelySecrets(redactHomePath(clipped))
|
||||
if (
|
||||
redacted !== clipped ||
|
||||
posix.isAbsolute(clipped) ||
|
||||
win32.isAbsolute(clipped) ||
|
||||
/(?:^|[^a-z0-9_])(?:[a-z]:[\\/]|[/\\]{1,2})/i.test(clipped)
|
||||
) {
|
||||
return '[redacted]'
|
||||
}
|
||||
return clipped
|
||||
}
|
||||
|
||||
function safeEventName(value: string): string {
|
||||
return /^[a-z0-9][a-z0-9._-]{0,79}$/i.test(value) ? value : 'unknown'
|
||||
}
|
||||
|
||||
function safeFiniteNumber(value: unknown): number | undefined {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : undefined
|
||||
}
|
||||
|
||||
const BUILTIN_ERROR_NAMES = new Set([
|
||||
'AggregateError',
|
||||
'Error',
|
||||
'EvalError',
|
||||
'RangeError',
|
||||
'ReferenceError',
|
||||
'SyntaxError',
|
||||
'TypeError',
|
||||
'URIError',
|
||||
])
|
||||
|
||||
const STANDARD_DOM_EXCEPTION_NAMES = new Set([
|
||||
'AbortError',
|
||||
'DataCloneError',
|
||||
'DataError',
|
||||
'EncodingError',
|
||||
'HierarchyRequestError',
|
||||
'InUseAttributeError',
|
||||
'IndexSizeError',
|
||||
'InvalidAccessError',
|
||||
'InvalidCharacterError',
|
||||
'InvalidModificationError',
|
||||
'InvalidNodeTypeError',
|
||||
'InvalidStateError',
|
||||
'NamespaceError',
|
||||
'NetworkError',
|
||||
'NoDataAllowedError',
|
||||
'NoModificationAllowedError',
|
||||
'NotAllowedError',
|
||||
'NotFoundError',
|
||||
'NotReadableError',
|
||||
'NotSupportedError',
|
||||
'OperationError',
|
||||
'QuotaExceededError',
|
||||
'ReadOnlyError',
|
||||
'SecurityError',
|
||||
'TimeoutError',
|
||||
'TransactionInactiveError',
|
||||
'UnknownError',
|
||||
'URLMismatchError',
|
||||
'VersionError',
|
||||
'WrongDocumentError',
|
||||
])
|
||||
|
||||
function getSafeErrorName(error: Error): string {
|
||||
if (error instanceof DOMException) {
|
||||
return STANDARD_DOM_EXCEPTION_NAMES.has(error.name)
|
||||
? `DOMException:${error.name}`
|
||||
: 'DOMException'
|
||||
}
|
||||
return BUILTIN_ERROR_NAMES.has(error.name) ? error.name : 'Error'
|
||||
}
|
||||
|
||||
function getRawReasonType(reason: unknown): string {
|
||||
if (reason === null) return 'null'
|
||||
if (reason instanceof DOMException) return getSafeErrorName(reason)
|
||||
if (reason instanceof Error) return `Error:${getSafeErrorName(reason)}`
|
||||
if (Array.isArray(reason)) return 'array'
|
||||
return typeof reason
|
||||
}
|
||||
|
||||
function getSafeErrorIdentity(error: unknown): string | undefined {
|
||||
if (error instanceof Error) return getSafeErrorName(error)
|
||||
return error === undefined ? undefined : typeof error
|
||||
}
|
||||
|
||||
function toSafeFields(fields: InterruptionTraceFields): SafeTraceFields {
|
||||
const reason = fields.reason
|
||||
const safe: SafeTraceFields = {}
|
||||
const stringFields = [
|
||||
'source',
|
||||
'subsystem',
|
||||
'phase',
|
||||
'queryId',
|
||||
'querySource',
|
||||
'parentQueryId',
|
||||
'subagentId',
|
||||
'providerRoute',
|
||||
'transport',
|
||||
'model',
|
||||
'attemptId',
|
||||
'controllerRole',
|
||||
'winningParentControllerId',
|
||||
'causalEventId',
|
||||
'trigger',
|
||||
'outcome',
|
||||
] as const
|
||||
for (const key of stringFields) {
|
||||
const value = safeString(fields[key])
|
||||
if (value !== undefined) safe[key] = value
|
||||
}
|
||||
const numberFields = [
|
||||
'queryGeneration',
|
||||
'elapsedQueryMs',
|
||||
'sinceLastActivityMs',
|
||||
'sinceLastRawByteMs',
|
||||
'sinceLastParsedFrameMs',
|
||||
'sinceLastYieldMs',
|
||||
'rawByteCount',
|
||||
'parsedFrameCount',
|
||||
'controlFrameCount',
|
||||
'ignoredFrameCount',
|
||||
'ignoredParsedFrameCount',
|
||||
'yieldedEventCount',
|
||||
'activeApiCallCount',
|
||||
'activeToolUseCount',
|
||||
'leaseCount',
|
||||
'suspendCount',
|
||||
'repeatedCount',
|
||||
'eventLoopDelayMaxMs',
|
||||
'eventLoopDelayMeanMs',
|
||||
] as const
|
||||
for (const key of numberFields) {
|
||||
const value = safeFiniteNumber(fields[key])
|
||||
if (value !== undefined) safe[key] = value
|
||||
}
|
||||
if (fields.parentControllerIds) {
|
||||
safe.parentControllerIds = fields.parentControllerIds
|
||||
.map(safeString)
|
||||
.filter((value): value is string => value !== undefined)
|
||||
.slice(0, 4)
|
||||
}
|
||||
if (reason !== undefined) {
|
||||
safe.normalizedReason = safeString(normalizeAbortReason(reason))
|
||||
safe.rawReasonType = getRawReasonType(reason)
|
||||
}
|
||||
if (fields.existingReason !== undefined) {
|
||||
safe.existingNormalizedReason = safeString(
|
||||
normalizeAbortReason(fields.existingReason),
|
||||
)
|
||||
safe.existingRawReasonType = getRawReasonType(fields.existingReason)
|
||||
}
|
||||
if (fields.attemptedReason !== undefined) {
|
||||
safe.attemptedNormalizedReason = safeString(
|
||||
normalizeAbortReason(fields.attemptedReason),
|
||||
)
|
||||
safe.attemptedRawReasonType = getRawReasonType(fields.attemptedReason)
|
||||
}
|
||||
const safeErrorIdentity = getSafeErrorIdentity(fields.error)
|
||||
if (safeErrorIdentity !== undefined) safe.safeErrorIdentity = safeErrorIdentity
|
||||
return safe
|
||||
}
|
||||
|
||||
function getAbortStackEvidence(): { abortStackFingerprint?: string } {
|
||||
const rawStack = new Error().stack
|
||||
if (!rawStack) return {}
|
||||
const sites = rawStack
|
||||
.split('\n')
|
||||
.slice(2, 7)
|
||||
.map(line => {
|
||||
const functionMatch = line.match(/^\s*at\s+([^\s(]+)/)
|
||||
const candidate = safeString(functionMatch?.[1])
|
||||
return candidate && !/[\\/:]/.test(candidate)
|
||||
? candidate
|
||||
: '<anonymous>'
|
||||
})
|
||||
const fingerprint = createHash('sha256').update(sites.join('>')).digest('hex').slice(0, 16)
|
||||
return { abortStackFingerprint: fingerprint }
|
||||
}
|
||||
|
||||
function buildEntry(
|
||||
event: string,
|
||||
fields: InterruptionTraceFields,
|
||||
nextSequence: number,
|
||||
extra: Partial<InterruptionTraceEntry> = {},
|
||||
allowDisabled = false,
|
||||
): InterruptionTraceEntry | undefined {
|
||||
if (!allowDisabled && !isEnabled()) return undefined
|
||||
if (!traceSessionId) traceSessionId = randomUUID()
|
||||
if (!eventLoopDelay) {
|
||||
eventLoopDelay = monitorEventLoopDelay({ resolution: 20 })
|
||||
eventLoopDelay.enable()
|
||||
}
|
||||
const monotonicMs = performance.now() - startedMonotonicMs
|
||||
const wallElapsedMs = Date.now() - startedWallMs
|
||||
const entry: InterruptionTraceEntry = {
|
||||
schemaVersion: TRACE_SCHEMA_VERSION,
|
||||
sequence: nextSequence,
|
||||
eventId: `${traceSessionId}:${nextSequence}`,
|
||||
traceSessionId,
|
||||
timestamp: new Date().toISOString(),
|
||||
monotonicMs,
|
||||
clockDeltaMs: wallElapsedMs - monotonicMs,
|
||||
event: safeEventName(event),
|
||||
...toSafeFields({
|
||||
...fields,
|
||||
eventLoopDelayMaxMs: Number.isFinite(eventLoopDelay.max)
|
||||
? eventLoopDelay.max / 1_000_000
|
||||
: 0,
|
||||
eventLoopDelayMeanMs: Number.isFinite(eventLoopDelay.mean)
|
||||
? eventLoopDelay.mean / 1_000_000
|
||||
: 0,
|
||||
}),
|
||||
...extra,
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
function addEntry(
|
||||
event: string,
|
||||
fields: InterruptionTraceFields,
|
||||
extra: Partial<InterruptionTraceEntry> = {},
|
||||
): InterruptionTraceEntry | undefined {
|
||||
if (!isEnabled()) return undefined
|
||||
try {
|
||||
const entry = buildEntry(event, fields, sequence + 1, extra)
|
||||
if (!entry) return undefined
|
||||
sequence = entry.sequence
|
||||
ring.push(entry)
|
||||
if (ring.length > TRACE_CAPACITY) ring = ring.slice(-TRACE_CAPACITY)
|
||||
return entry
|
||||
} catch {
|
||||
// Diagnostics must be total for arbitrary abort reasons and metadata.
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
export function traceInterruptionEvent(
|
||||
event: string,
|
||||
fields: InterruptionTraceFields = {},
|
||||
): string | undefined {
|
||||
return addEntry(event, fields)?.eventId
|
||||
}
|
||||
|
||||
export function setInterruptionErrorCausalEventId(
|
||||
error: unknown,
|
||||
eventId: string | undefined,
|
||||
): void {
|
||||
if (
|
||||
eventId &&
|
||||
((typeof error === 'object' && error !== null) ||
|
||||
typeof error === 'function')
|
||||
) {
|
||||
errorCausalEventIds.set(error, eventId)
|
||||
}
|
||||
}
|
||||
|
||||
export function getInterruptionErrorCausalEventId(
|
||||
error: unknown,
|
||||
): string | undefined {
|
||||
return (typeof error === 'object' && error !== null) ||
|
||||
typeof error === 'function'
|
||||
? errorCausalEventIds.get(error)
|
||||
: undefined
|
||||
}
|
||||
|
||||
export function tracePermissionAbortResolution(
|
||||
source: string | undefined,
|
||||
causalEventId: string | undefined,
|
||||
subsystem: string,
|
||||
): string | undefined {
|
||||
return traceInterruptionEvent('permission.abort_resolved', {
|
||||
source: source ?? 'unknown',
|
||||
subsystem,
|
||||
causalEventId,
|
||||
outcome: 'denied',
|
||||
})
|
||||
}
|
||||
|
||||
export function registerInterruptionController(
|
||||
controller: AbortController,
|
||||
fields: InterruptionTraceFields = {},
|
||||
options: {
|
||||
provisionalRole?: boolean
|
||||
refreshQueryContext?: boolean
|
||||
} = {},
|
||||
): string | undefined {
|
||||
if (!isEnabled()) return undefined
|
||||
const existing = controllerStates.get(controller)
|
||||
if (existing) {
|
||||
// Registration metadata describes controller identity. Later callers may
|
||||
// add missing context, but must not relabel an established query root as a
|
||||
// parent/tool/controller role merely because they are observing it there.
|
||||
existing.fields = { ...fields, ...existing.fields }
|
||||
if (
|
||||
existing.provisionalRole &&
|
||||
fields.controllerRole !== undefined &&
|
||||
!options.provisionalRole
|
||||
) {
|
||||
existing.fields.controllerRole = fields.controllerRole
|
||||
existing.provisionalRole = false
|
||||
}
|
||||
if (options.refreshQueryContext) {
|
||||
existing.fields = {
|
||||
...existing.fields,
|
||||
...(fields.queryId !== undefined && { queryId: fields.queryId }),
|
||||
...(fields.queryGeneration !== undefined && {
|
||||
queryGeneration: fields.queryGeneration,
|
||||
}),
|
||||
...(fields.querySource !== undefined && {
|
||||
querySource: fields.querySource,
|
||||
}),
|
||||
...(fields.parentQueryId !== undefined && {
|
||||
parentQueryId: fields.parentQueryId,
|
||||
}),
|
||||
}
|
||||
}
|
||||
return existing.id
|
||||
}
|
||||
|
||||
const id = `controller-${++controllerCounter}`
|
||||
controllerStates.set(controller, {
|
||||
id,
|
||||
repeatedCount: 0,
|
||||
fields,
|
||||
provisionalRole: options.provisionalRole === true,
|
||||
})
|
||||
signalIds.set(controller.signal, id)
|
||||
addEntry('controller.registered', fields, { controllerId: id })
|
||||
const observeAbort = () => {
|
||||
const state = controllerStates.get(controller)
|
||||
const observed = addEntry(
|
||||
'signal.observed',
|
||||
{ ...state?.fields, reason: controller.signal.reason },
|
||||
{
|
||||
controllerId: id,
|
||||
...(state?.firstAbortEventId && {
|
||||
firstAbortEventId: state.firstAbortEventId,
|
||||
}),
|
||||
},
|
||||
)
|
||||
if (observed && !signalAbortEventIds.has(controller.signal)) {
|
||||
signalAbortEventIds.set(controller.signal, observed.eventId)
|
||||
}
|
||||
if (
|
||||
observed &&
|
||||
state?.fields.source &&
|
||||
!signalAbortSources.has(controller.signal)
|
||||
) {
|
||||
signalAbortSources.set(controller.signal, state.fields.source)
|
||||
}
|
||||
if (observed && state && !state.firstAbortEventId) {
|
||||
state.firstAbortEventId = observed.eventId
|
||||
}
|
||||
if (state?.fields.controllerRole === 'query-root') {
|
||||
flushInterruptionTrace('root_abort_observed')
|
||||
}
|
||||
}
|
||||
if (controller.signal.aborted) observeAbort()
|
||||
else controller.signal.addEventListener('abort', observeAbort, { once: true })
|
||||
return id
|
||||
}
|
||||
|
||||
function preserveControllerIdentity(
|
||||
fields: InterruptionTraceFields,
|
||||
state: ControllerTraceState | undefined,
|
||||
): InterruptionTraceFields {
|
||||
const identity = state?.fields
|
||||
if (!identity) return fields
|
||||
return {
|
||||
...fields,
|
||||
...(identity.queryId !== undefined && { queryId: identity.queryId }),
|
||||
...(identity.queryGeneration !== undefined && {
|
||||
queryGeneration: identity.queryGeneration,
|
||||
}),
|
||||
...(identity.querySource !== undefined && {
|
||||
querySource: identity.querySource,
|
||||
}),
|
||||
...(identity.parentQueryId !== undefined && {
|
||||
parentQueryId: identity.parentQueryId,
|
||||
}),
|
||||
...(identity.subagentId !== undefined && {
|
||||
subagentId: identity.subagentId,
|
||||
}),
|
||||
...(identity.controllerRole !== undefined && {
|
||||
controllerRole: identity.controllerRole,
|
||||
}),
|
||||
...(identity.parentControllerIds !== undefined && {
|
||||
parentControllerIds: identity.parentControllerIds,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
export function getInterruptionSignalId(signal: AbortSignal): string | undefined {
|
||||
return isEnabled() ? signalIds.get(signal) : undefined
|
||||
}
|
||||
|
||||
export function getInterruptionSignalAbortEventId(
|
||||
signal: AbortSignal,
|
||||
): string | undefined {
|
||||
return isEnabled() ? signalAbortEventIds.get(signal) : undefined
|
||||
}
|
||||
|
||||
export function getInterruptionSignalAbortTrace(
|
||||
signal: AbortSignal,
|
||||
): { source?: string; causalEventId?: string } {
|
||||
if (!isEnabled()) return {}
|
||||
return {
|
||||
source: signalAbortSources.get(signal),
|
||||
causalEventId: signalAbortEventIds.get(signal),
|
||||
}
|
||||
}
|
||||
|
||||
export function registerInterruptionSignal(
|
||||
signal: AbortSignal,
|
||||
fields: InterruptionTraceFields = {},
|
||||
getAbortFields?: () => InterruptionTraceFields,
|
||||
): string | undefined {
|
||||
if (!isEnabled()) return undefined
|
||||
const existing = signalIds.get(signal)
|
||||
if (existing) return existing
|
||||
const id = `signal-${++signalCounter}`
|
||||
signalIds.set(signal, id)
|
||||
signalStates.set(signal, { id, fields, getAbortFields })
|
||||
addEntry('signal.registered', fields, { controllerId: id })
|
||||
const observeAbort = () => observeRegisteredSignal(signal)
|
||||
if (signal.aborted) observeAbort()
|
||||
else signal.addEventListener('abort', observeAbort, { once: true })
|
||||
return id
|
||||
}
|
||||
|
||||
function observeRegisteredSignal(signal: AbortSignal): string | undefined {
|
||||
const existingEventId = signalAbortEventIds.get(signal)
|
||||
if (existingEventId) return existingEventId
|
||||
const state = signalStates.get(signal)
|
||||
if (!state) return undefined
|
||||
const observed = addEntry(
|
||||
'signal.observed',
|
||||
{ ...state.fields, ...state.getAbortFields?.(), reason: signal.reason },
|
||||
{ controllerId: state.id },
|
||||
)
|
||||
if (observed) signalAbortEventIds.set(signal, observed.eventId)
|
||||
return observed?.eventId
|
||||
}
|
||||
|
||||
export function traceCombinedAbortSignal(
|
||||
combinedSignal: AbortSignal,
|
||||
parents: readonly AbortSignal[],
|
||||
fields: InterruptionTraceFields = {},
|
||||
): string | undefined {
|
||||
if (!isEnabled()) return undefined
|
||||
const parentControllerIds = parents
|
||||
.map(parent =>
|
||||
registerInterruptionSignal(parent, {
|
||||
subsystem: fields.subsystem,
|
||||
controllerRole: 'combined-parent',
|
||||
}),
|
||||
)
|
||||
.filter((value): value is string => value !== undefined)
|
||||
return registerInterruptionSignal(
|
||||
combinedSignal,
|
||||
{
|
||||
...fields,
|
||||
parentControllerIds,
|
||||
},
|
||||
() => {
|
||||
const winner = parents.find(parent => parent.aborted)
|
||||
if (!winner) return {}
|
||||
// AbortSignal.any can abort the combined signal before ordinary listeners
|
||||
// on the winning parent run. Record the parent synchronously here so the
|
||||
// combined observation carries the causal edge on the native path.
|
||||
observeRegisteredSignal(winner)
|
||||
return {
|
||||
winningParentControllerId: getInterruptionSignalId(winner),
|
||||
causalEventId: getInterruptionSignalAbortEventId(winner),
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
export function requestAbort(
|
||||
controller: AbortController,
|
||||
reason: unknown,
|
||||
fields: InterruptionTraceFields,
|
||||
): void {
|
||||
if (!isEnabled()) {
|
||||
controller.abort(reason)
|
||||
return
|
||||
}
|
||||
|
||||
let shouldFlushRoot = fields.controllerRole === 'query-root'
|
||||
try {
|
||||
const controllerId =
|
||||
registerInterruptionController(controller, fields) ?? 'controller-unknown'
|
||||
const state = controllerStates.get(controller)
|
||||
const requestFields = preserveControllerIdentity(fields, state)
|
||||
shouldFlushRoot ||= state?.fields.controllerRole === 'query-root'
|
||||
if (controller.signal.aborted) {
|
||||
if (state) state.repeatedCount++
|
||||
addEntry(
|
||||
'abort.repeated',
|
||||
{
|
||||
...requestFields,
|
||||
existingReason: controller.signal.reason,
|
||||
attemptedReason: reason,
|
||||
outcome: 'ignored_first_abort_wins',
|
||||
repeatedCount: state?.repeatedCount ?? 1,
|
||||
},
|
||||
{
|
||||
controllerId,
|
||||
...(state?.firstAbortEventId && {
|
||||
firstAbortEventId: state.firstAbortEventId,
|
||||
}),
|
||||
},
|
||||
)
|
||||
} else {
|
||||
const entry = addEntry(
|
||||
'abort.requested',
|
||||
{ ...requestFields, reason },
|
||||
{ controllerId, ...getAbortStackEvidence() },
|
||||
)
|
||||
if (entry && state) {
|
||||
state.firstAbortEventId = entry.eventId
|
||||
signalAbortEventIds.set(controller.signal, entry.eventId)
|
||||
if (fields.source) {
|
||||
signalAbortSources.set(controller.signal, fields.source)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Best-effort diagnostics must never interfere with the native abort.
|
||||
} finally {
|
||||
controller.abort(reason)
|
||||
}
|
||||
if (shouldFlushRoot) flushInterruptionTrace('root_abort_observed')
|
||||
}
|
||||
|
||||
export function traceCombinedSignal(
|
||||
combinedController: AbortController,
|
||||
parents: readonly (AbortSignal | undefined)[],
|
||||
fields: InterruptionTraceFields = {},
|
||||
): string | undefined {
|
||||
if (!isEnabled()) return undefined
|
||||
const parentControllerIds = parents
|
||||
.map(parent =>
|
||||
parent
|
||||
? registerInterruptionSignal(parent, {
|
||||
subsystem: fields.subsystem,
|
||||
controllerRole: 'combined-parent',
|
||||
})
|
||||
: undefined,
|
||||
)
|
||||
.filter((value): value is string => value !== undefined)
|
||||
return registerInterruptionController(combinedController, {
|
||||
...fields,
|
||||
parentControllerIds,
|
||||
})
|
||||
}
|
||||
|
||||
async function performInterruptionTraceFlush(
|
||||
trigger: string,
|
||||
logFile: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
while (true) {
|
||||
let batch = retryBatch
|
||||
if (!batch) {
|
||||
const pending = ring.filter(
|
||||
entry => entry.sequence > flushedThroughSequence,
|
||||
)
|
||||
if (pending.length === 0) return
|
||||
const marker = buildEntry(
|
||||
'trace.flush',
|
||||
{ trigger, repeatedCount: pending.length },
|
||||
sequence + 1,
|
||||
{},
|
||||
true,
|
||||
)
|
||||
if (!marker) return
|
||||
// Reserve the marker before awaiting. Later events get larger IDs, and
|
||||
// this immutable batch remains reachable even if the ring wraps.
|
||||
sequence = marker.sequence
|
||||
batch = { entries: [...pending, marker], marker, logFile }
|
||||
}
|
||||
|
||||
const result = await appendDiagnosticsNoPII(batch.logFile, batch.entries)
|
||||
if (result === 'retryable_failure') {
|
||||
retryBatch = batch
|
||||
return
|
||||
}
|
||||
|
||||
retryBatch = undefined
|
||||
flushedThroughSequence = batch.marker.sequence
|
||||
if (result === 'committed') {
|
||||
ring = [...ring, batch.marker]
|
||||
.sort((left, right) => left.sequence - right.sequence)
|
||||
.slice(-TRACE_CAPACITY)
|
||||
}
|
||||
// Keep draining: events can be recorded while an append is in flight,
|
||||
// including during graceful shutdown after the caller requested a flush.
|
||||
}
|
||||
} catch {
|
||||
// A trace flush is never allowed to affect request cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
export function flushInterruptionTrace(trigger: string): void {
|
||||
if (!isEnabled()) return
|
||||
const logFile = process.env[TRACE_FILE_ENV]
|
||||
if (!logFile || !isAbsolute(logFile)) return
|
||||
flushQueue = flushQueue
|
||||
.then(() => performInterruptionTraceFlush(trigger, logFile))
|
||||
.catch(() => {
|
||||
// Diagnostics are deliberately detached from request cancellation.
|
||||
})
|
||||
}
|
||||
|
||||
export async function waitForInterruptionTraceFlush(): Promise<void> {
|
||||
await flushQueue
|
||||
}
|
||||
|
||||
export async function __waitForInterruptionTraceFlushForTests(): Promise<void> {
|
||||
await waitForInterruptionTraceFlush()
|
||||
}
|
||||
|
||||
export function __getInterruptionTraceSnapshotForTests(): readonly InterruptionTraceEntry[] {
|
||||
return [...ring]
|
||||
}
|
||||
|
||||
export function __resetInterruptionTraceForTests(): void {
|
||||
traceSessionId = ''
|
||||
sequence = 0
|
||||
startedWallMs = Date.now()
|
||||
startedMonotonicMs = performance.now()
|
||||
ring = []
|
||||
flushedThroughSequence = 0
|
||||
controllerCounter = 0
|
||||
signalCounter = 0
|
||||
controllerStates = new WeakMap()
|
||||
signalIds = new WeakMap()
|
||||
signalAbortEventIds = new WeakMap()
|
||||
signalAbortSources = new WeakMap()
|
||||
signalStates = new WeakMap()
|
||||
errorCausalEventIds = new WeakMap()
|
||||
retryBatch = undefined
|
||||
flushQueue = Promise.resolve()
|
||||
eventLoopDelay?.disable()
|
||||
eventLoopDelay = undefined
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import type { Tool, ToolUseContext } from 'src/Tool.js'
|
||||
import z from 'zod/v4'
|
||||
import { logForDebugging } from '../debug.js'
|
||||
import { lazySchema } from '../lazySchema.js'
|
||||
import { requestAbort } from '../interruptionTrace.js'
|
||||
import type {
|
||||
PermissionDecision,
|
||||
PermissionDecisionReason,
|
||||
@@ -159,7 +160,11 @@ export async function permissionPromptToolResultToPermissionDecision(
|
||||
logForDebugging(
|
||||
`SDK permission prompt deny+interrupt: tool=${tool.name} message=${result.message}`,
|
||||
)
|
||||
toolUseContext.abortController.abort('interrupt')
|
||||
requestAbort(toolUseContext.abortController, 'interrupt', {
|
||||
source: 'sdk_permission_interrupt',
|
||||
subsystem: 'tool_permission',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
return {
|
||||
...result,
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
import { afterAll, beforeAll, describe, expect, mock, test } from 'bun:test'
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
describe,
|
||||
expect,
|
||||
mock,
|
||||
test,
|
||||
} from 'bun:test'
|
||||
import { z } from 'zod/v4'
|
||||
import type { ToolPermissionContext, ToolUseContext } from '../../Tool.js'
|
||||
import { createToolFixture } from '../../test/toolFixtures.js'
|
||||
@@ -6,13 +14,15 @@ import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../../test/sharedMutationLock.js'
|
||||
import type { PermissionDecision } from './PermissionResult.js'
|
||||
import type { PermissionRequestResult } from '../../types/hooks.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
} from '../interruptionTrace.js'
|
||||
import { permissionPromptToolResultToPermissionDecision } from './PermissionPromptToolResultSchema.js'
|
||||
import type { PermissionUpdate } from './PermissionUpdateSchema.js'
|
||||
|
||||
type HookDecision = PermissionDecision & {
|
||||
updatedPermissions?: PermissionUpdate[]
|
||||
}
|
||||
type HookDecision = PermissionRequestResult
|
||||
|
||||
let hookDecision: HookDecision
|
||||
let hasPermissionsToUseTool: typeof import('./permissions.js').hasPermissionsToUseTool
|
||||
@@ -20,6 +30,7 @@ let createPermissionContext: typeof import('../../hooks/toolPermission/Permissio
|
||||
let StructuredIO: typeof import('../../cli/structuredIO.js').StructuredIO
|
||||
let actualHooks: typeof import('../hooks.js')
|
||||
let beforeHookDecision: (() => void) | undefined
|
||||
const originalInterruptionTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
|
||||
beforeAll(async () => {
|
||||
await acquireSharedMutationLock(
|
||||
@@ -55,6 +66,16 @@ afterAll(() => {
|
||||
}
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalInterruptionTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalInterruptionTrace
|
||||
}
|
||||
})
|
||||
|
||||
function planContext(
|
||||
overrides: Partial<ToolPermissionContext> = {},
|
||||
): {
|
||||
@@ -367,6 +388,82 @@ describe('headless plan-mode PermissionRequest hooks', () => {
|
||||
expect(state.getPermissionContext().mode).toBe('plan')
|
||||
})
|
||||
|
||||
test('labels SDK permission prompt interrupts as query-root aborts', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const tool = createToolFixture(z.object({}), {
|
||||
name: 'SDKPromptInterruptTool',
|
||||
isReadOnly: () => true,
|
||||
})
|
||||
const state = planContext({ mode: 'default' })
|
||||
|
||||
const result = await permissionPromptToolResultToPermissionDecision(
|
||||
{
|
||||
behavior: 'deny',
|
||||
message: 'Stop the query',
|
||||
interrupt: true,
|
||||
},
|
||||
tool,
|
||||
{},
|
||||
state.context,
|
||||
)
|
||||
|
||||
expect(result.behavior).toBe('deny')
|
||||
expect(state.context.abortController.signal.aborted).toBe(true)
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.source === 'sdk_permission_interrupt',
|
||||
),
|
||||
).toMatchObject({
|
||||
subsystem: 'tool_permission',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
})
|
||||
|
||||
test('labels headless permission hook interrupts as query-root aborts', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const tool = createToolFixture(z.object({}), {
|
||||
name: 'HeadlessHookInterruptTool',
|
||||
isReadOnly: () => true,
|
||||
async checkPermissions() {
|
||||
return { behavior: 'ask' as const, message: 'Review read' }
|
||||
},
|
||||
})
|
||||
const state = planContext({
|
||||
mode: 'default',
|
||||
shouldAvoidPermissionPrompts: true,
|
||||
})
|
||||
hookDecision = {
|
||||
behavior: 'deny',
|
||||
message: 'Stop the query',
|
||||
interrupt: true,
|
||||
}
|
||||
|
||||
const result = await hasPermissionsToUseTool(
|
||||
tool,
|
||||
{},
|
||||
state.context,
|
||||
assistantMessage,
|
||||
'headless-hook-interrupt',
|
||||
)
|
||||
|
||||
expect(result.behavior).toBe('deny')
|
||||
expect(state.context.abortController.signal.aborted).toBe(true)
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.source === 'permission_hook',
|
||||
),
|
||||
).toMatchObject({
|
||||
subsystem: 'tool_permission',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
})
|
||||
|
||||
test('SDK permission prompt rechecks plan mode after approval normalization', async () => {
|
||||
const conditionalTool = createToolFixture(
|
||||
z.object({ operation: z.enum(['read', 'write']) }),
|
||||
@@ -597,6 +694,47 @@ describe('headless plan-mode PermissionRequest hooks', () => {
|
||||
])
|
||||
})
|
||||
|
||||
test('labels interactive permission hook interrupts as query-root aborts', async () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
const tool = createToolFixture(z.object({}), {
|
||||
name: 'InteractiveHookInterruptTool',
|
||||
isReadOnly: () => true,
|
||||
})
|
||||
const state = planContext({ mode: 'default' })
|
||||
const permissionContext = createPermissionContext(
|
||||
tool,
|
||||
{},
|
||||
state.context,
|
||||
{ message: { id: 'assistant-message' } } as never,
|
||||
'interactive-hook-interrupt',
|
||||
state.setPermissionContext,
|
||||
)
|
||||
hookDecision = {
|
||||
behavior: 'deny',
|
||||
message: 'Stop the query',
|
||||
interrupt: true,
|
||||
}
|
||||
|
||||
const result = await permissionContext.runHooks(undefined, undefined)
|
||||
|
||||
expect(result).toMatchObject({
|
||||
behavior: 'deny',
|
||||
message: 'Stop the query',
|
||||
})
|
||||
expect(state.context.abortController.signal.aborted).toBe(true)
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry =>
|
||||
entry.event === 'abort.requested' &&
|
||||
entry.source === 'permission_hook',
|
||||
),
|
||||
).toMatchObject({
|
||||
subsystem: 'tool_permission',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
})
|
||||
|
||||
test('interactive PermissionRequest hooks cannot rewrite a read into a mutation', async () => {
|
||||
const conditionalTool = createToolFixture(
|
||||
z.object({ operation: z.enum(['read', 'write']) }),
|
||||
|
||||
@@ -25,6 +25,7 @@ import { extractOutputRedirections } from '../bash/commands.js'
|
||||
import { logForDebugging } from '../debug.js'
|
||||
import { AbortError, toError } from '../errors.js'
|
||||
import { logError } from '../log.js'
|
||||
import { requestAbort } from '../interruptionTrace.js'
|
||||
import { SandboxManager } from '../sandbox/sandbox-adapter.js'
|
||||
import {
|
||||
getSettingSourceDisplayNameLowercase,
|
||||
@@ -524,7 +525,11 @@ async function runPermissionRequestHooksForHeadlessAgent(
|
||||
logForDebugging(
|
||||
`Hook interrupt: tool=${tool.name} hookMessage=${decision.message}`,
|
||||
)
|
||||
context.abortController.abort('interrupt')
|
||||
requestAbort(context.abortController, 'interrupt', {
|
||||
source: 'permission_hook',
|
||||
subsystem: 'tool_permission',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
return {
|
||||
behavior: 'deny',
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { driveQueryEvents } from './queryEventDriver.js'
|
||||
|
||||
describe('driveQueryEvents', () => {
|
||||
test('registers activity only for yielded events and returns terminal state', async () => {
|
||||
async function* query(): AsyncGenerator<
|
||||
{ type: string },
|
||||
{ reason: string }
|
||||
> {
|
||||
yield { type: 'stream_request_start' }
|
||||
yield { type: 'stream_event' }
|
||||
return { reason: 'completed' }
|
||||
}
|
||||
const activity: string[] = []
|
||||
const events: string[] = []
|
||||
|
||||
const terminal = await driveQueryEvents(
|
||||
query(),
|
||||
reason => activity.push(reason),
|
||||
event => events.push(event.type),
|
||||
)
|
||||
|
||||
expect(activity).toEqual([
|
||||
'query_event:stream_request_start',
|
||||
'query_event:stream_event',
|
||||
])
|
||||
expect(events).toEqual(['stream_request_start', 'stream_event'])
|
||||
expect(terminal).toEqual({ reason: 'completed' })
|
||||
})
|
||||
|
||||
test('closes the generator if event handling throws', async () => {
|
||||
let finalized = false
|
||||
async function* query(): AsyncGenerator<{ type: string }, void> {
|
||||
try {
|
||||
yield { type: 'stream_event' }
|
||||
yield { type: 'must_not_be_seen' }
|
||||
} finally {
|
||||
finalized = true
|
||||
}
|
||||
}
|
||||
|
||||
await expect(
|
||||
driveQueryEvents(
|
||||
query(),
|
||||
() => {},
|
||||
() => {
|
||||
throw new Error('consumer failed')
|
||||
},
|
||||
),
|
||||
).rejects.toThrow('consumer failed')
|
||||
expect(finalized).toBe(true)
|
||||
})
|
||||
|
||||
test('does not let generator cleanup mask an event-handler failure', async () => {
|
||||
const generator: AsyncGenerator<{ type: string }, void> = {
|
||||
next: async () => ({ done: false, value: { type: 'stream_event' } }),
|
||||
return: async () => {
|
||||
throw new Error('generator cleanup failed')
|
||||
},
|
||||
throw: async error => {
|
||||
throw error
|
||||
},
|
||||
[Symbol.asyncIterator]() {
|
||||
return this
|
||||
},
|
||||
[Symbol.asyncDispose]: async () => {},
|
||||
}
|
||||
|
||||
await expect(
|
||||
driveQueryEvents(
|
||||
generator,
|
||||
() => {},
|
||||
() => {
|
||||
throw new Error('consumer failed')
|
||||
},
|
||||
),
|
||||
).rejects.toThrow('consumer failed')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* Consume a query event generator while preserving the REPL's activity policy:
|
||||
* only events yielded to the consumer count as QueryGuard progress.
|
||||
*/
|
||||
export async function driveQueryEvents<TEvent extends { type: string }, TReturn>(
|
||||
queryGenerator: AsyncGenerator<TEvent, TReturn>,
|
||||
registerActivity: (reason: string) => void,
|
||||
onEvent: (event: TEvent) => void,
|
||||
): Promise<TReturn> {
|
||||
let generatorDone = false
|
||||
try {
|
||||
while (true) {
|
||||
const next = await queryGenerator.next()
|
||||
if (next.done) {
|
||||
generatorDone = true
|
||||
return next.value
|
||||
}
|
||||
registerActivity(`query_event:${next.value.type}`)
|
||||
onEvent(next.value)
|
||||
}
|
||||
} finally {
|
||||
if (!generatorDone) {
|
||||
try {
|
||||
await queryGenerator.return(undefined as never)
|
||||
} catch {
|
||||
// Preserve the generator or event-handler failure that triggered cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, test } from 'vitest'
|
||||
import {
|
||||
formatQueryLifecycleAbortSignalReason,
|
||||
formatQueryLifecycleLogMessage,
|
||||
getQueryTerminalOutcome,
|
||||
getQueryTerminalReason,
|
||||
type QueryLifecycleContext,
|
||||
} from './queryLifecycle.js'
|
||||
@@ -47,6 +48,13 @@ describe('query lifecycle log formatting', () => {
|
||||
})
|
||||
|
||||
describe('query terminal reason classification', () => {
|
||||
test('classifies coarse terminal outcomes with abort precedence', () => {
|
||||
expect(getQueryTerminalOutcome({ aborted: false }, false)).toBe('completed')
|
||||
expect(getQueryTerminalOutcome({ aborted: false }, true)).toBe('failed')
|
||||
expect(getQueryTerminalOutcome({ aborted: true }, false)).toBe('aborted')
|
||||
expect(getQueryTerminalOutcome({ aborted: true }, true)).toBe('aborted')
|
||||
})
|
||||
|
||||
test('classifies non-aborted completion from throw state', () => {
|
||||
expect(
|
||||
getQueryTerminalReason({ aborted: false, reason: undefined }, false),
|
||||
|
||||
@@ -13,6 +13,8 @@ export type QueryTerminalReason =
|
||||
|
||||
export type QueryGuardTimeoutReason = 'idle' | 'hard_max' | 'lease_expired'
|
||||
|
||||
export type QueryTerminalOutcome = 'aborted' | 'failed' | 'completed'
|
||||
|
||||
export type QueryActiveApiCall = {
|
||||
clientRequestId?: string
|
||||
requestId?: string | null
|
||||
@@ -67,6 +69,7 @@ export type QueryGuardTimeoutInfo = {
|
||||
elapsedMs: number
|
||||
context: QueryLifecycleContext
|
||||
activeOperations: QueryActiveOperationSnapshot
|
||||
causalEventId?: string
|
||||
}
|
||||
|
||||
export function getQueryTerminalReason(
|
||||
@@ -93,6 +96,14 @@ export function getQueryTerminalReason(
|
||||
}
|
||||
}
|
||||
|
||||
export function getQueryTerminalOutcome(
|
||||
signal: Pick<AbortSignal, 'aborted'>,
|
||||
didThrow: boolean,
|
||||
): QueryTerminalOutcome {
|
||||
if (signal.aborted) return 'aborted'
|
||||
return didThrow ? 'failed' : 'completed'
|
||||
}
|
||||
|
||||
export function formatQueryLifecycleAbortSignalReason(reason: string): string {
|
||||
return `abortSignalReason=${reason}`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
} from './interruptionTrace.js'
|
||||
import {
|
||||
requestBackgroundHandoffAbort,
|
||||
requestBridgeInterrupt,
|
||||
requestPriorityNowAbort,
|
||||
} from './replInterruption.js'
|
||||
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
|
||||
beforeEach(async () => {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
else process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
})
|
||||
|
||||
describe('REPL interruption source wiring', () => {
|
||||
test('bridge interruption reads the current controller and links its input event', () => {
|
||||
const stale = new AbortController()
|
||||
const current = new AbortController()
|
||||
const ref = { current: stale }
|
||||
ref.current = current
|
||||
|
||||
requestBridgeInterrupt(ref)
|
||||
|
||||
expect(stale.signal.aborted).toBe(false)
|
||||
expect(current.signal.aborted).toBe(true)
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const input = trace.find(entry => entry.event === 'input.bridge_interrupt')
|
||||
const abort = trace.find(entry => entry.event === 'abort.requested')
|
||||
expect(abort?.source).toBe('bridge_interrupt')
|
||||
expect(input).toBeDefined()
|
||||
expect(abort).toBeDefined()
|
||||
expect(typeof input!.eventId).toBe('string')
|
||||
expect(typeof abort!.causalEventId).toBe('string')
|
||||
expect(abort!.causalEventId).toBe(input!.eventId)
|
||||
})
|
||||
|
||||
test('background handoff aborts the foreground query with its source', () => {
|
||||
const controller = new AbortController()
|
||||
|
||||
requestBackgroundHandoffAbort(controller)
|
||||
|
||||
expect(controller.signal.reason).toBe('background')
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'abort.requested',
|
||||
)?.source,
|
||||
).toBe('background_handoff')
|
||||
})
|
||||
|
||||
test('priority-now reads and aborts the current query controller', () => {
|
||||
const stale = new AbortController()
|
||||
const current = new AbortController()
|
||||
const ref = { current: stale }
|
||||
ref.current = current
|
||||
|
||||
requestPriorityNowAbort(ref)
|
||||
|
||||
expect(stale.signal.aborted).toBe(false)
|
||||
expect(current.signal.reason).toBe('interrupt')
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'abort.requested',
|
||||
)?.source,
|
||||
).toBe('priority_now')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,39 @@
|
||||
import { requestAbort, traceInterruptionEvent } from './interruptionTrace.js'
|
||||
|
||||
type AbortControllerRef = { readonly current: AbortController | null }
|
||||
|
||||
export function requestBridgeInterrupt(ref: AbortControllerRef): void {
|
||||
const controller = ref.current
|
||||
const causalEventId = traceInterruptionEvent('input.bridge_interrupt', {
|
||||
source: 'bridge_interrupt',
|
||||
subsystem: 'repl_bridge',
|
||||
})
|
||||
if (!controller) return
|
||||
requestAbort(controller, 'interrupt', {
|
||||
source: 'bridge_interrupt',
|
||||
subsystem: 'repl_bridge',
|
||||
controllerRole: 'query-root',
|
||||
causalEventId,
|
||||
})
|
||||
}
|
||||
|
||||
export function requestBackgroundHandoffAbort(
|
||||
controller: AbortController | null,
|
||||
): void {
|
||||
if (!controller) return
|
||||
requestAbort(controller, 'background', {
|
||||
source: 'background_handoff',
|
||||
subsystem: 'repl',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
|
||||
export function requestPriorityNowAbort(ref: AbortControllerRef): void {
|
||||
const controller = ref.current
|
||||
if (!controller) return
|
||||
requestAbort(controller, 'interrupt', {
|
||||
source: 'priority_now',
|
||||
subsystem: 'repl',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import { afterEach, beforeEach, expect, test } from 'bun:test'
|
||||
import type { ToolUseConfirm } from '../../components/permissions/PermissionRequest.js'
|
||||
import { getDefaultAppState } from '../../state/AppStateStore.js'
|
||||
import type { Tool } from '../../Tool.js'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../../test/sharedMutationLock.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
__waitForInterruptionTraceFlushForTests,
|
||||
requestAbort,
|
||||
} from '../interruptionTrace.js'
|
||||
import { createInProcessPermissionAbortCompleter } from './inProcessPermissionAbort.js'
|
||||
import { createInProcessCanUseTool } from './inProcessRunner.js'
|
||||
import {
|
||||
registerLeaderToolUseConfirmQueue,
|
||||
unregisterLeaderToolUseConfirmQueue,
|
||||
} from './leaderPermissionBridge.js'
|
||||
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock('inProcessPermissionAbort.test.ts')
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
__resetInterruptionTraceForTests()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await __waitForInterruptionTraceFlushForTests()
|
||||
__resetInterruptionTraceForTests()
|
||||
unregisterLeaderToolUseConfirmQueue()
|
||||
if (originalTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
} finally {
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
test('links signal-driven permission denial to the winning abort request once', () => {
|
||||
const controller = new AbortController()
|
||||
let deniedCount = 0
|
||||
const completion = createInProcessPermissionAbortCompleter(
|
||||
controller.signal,
|
||||
() => {
|
||||
deniedCount++
|
||||
},
|
||||
)
|
||||
|
||||
requestAbort(controller, undefined, {
|
||||
source: 'query_guard',
|
||||
subsystem: 'query_engine',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const abortRequest = trace.find(entry => entry.event === 'abort.requested')
|
||||
const permissionResolution = trace.find(
|
||||
entry => entry.event === 'permission.abort_resolved',
|
||||
)
|
||||
expect(deniedCount).toBe(1)
|
||||
expect(abortRequest?.eventId).toBeString()
|
||||
expect(permissionResolution).toMatchObject({
|
||||
source: 'query_guard',
|
||||
subsystem: 'in_process_permission_bridge',
|
||||
outcome: 'denied',
|
||||
causalEventId: abortRequest?.eventId,
|
||||
})
|
||||
expect(completion.completeAbort('ui_cancel', 'later-event')).toBe(false)
|
||||
expect(deniedCount).toBe(1)
|
||||
})
|
||||
|
||||
test('aborts a queued in-process permission through the production signal path', async () => {
|
||||
const controller = new AbortController()
|
||||
let queue: ToolUseConfirm[] = []
|
||||
registerLeaderToolUseConfirmQueue(updater => {
|
||||
queue = updater(queue)
|
||||
})
|
||||
const canUseTool = createInProcessCanUseTool(
|
||||
{
|
||||
agentId: 'worker-1',
|
||||
agentName: 'worker',
|
||||
teamName: 'test-team',
|
||||
planModeRequired: false,
|
||||
parentSessionId: 'parent-session',
|
||||
},
|
||||
controller,
|
||||
)
|
||||
const appState = getDefaultAppState()
|
||||
const decisionPromise = canUseTool(
|
||||
{
|
||||
name: 'TraceTestTool',
|
||||
description: async () => 'test permission',
|
||||
} as unknown as Tool,
|
||||
{},
|
||||
{
|
||||
getAppState: () => appState,
|
||||
options: {
|
||||
isNonInteractiveSession: false,
|
||||
tools: [],
|
||||
},
|
||||
} as never,
|
||||
{} as never,
|
||||
'tool-use-1',
|
||||
{ behavior: 'ask', message: 'approval required' },
|
||||
)
|
||||
|
||||
for (let attempt = 0; attempt < 10 && queue.length === 0; attempt++) {
|
||||
await Promise.resolve()
|
||||
}
|
||||
expect(queue).toHaveLength(1)
|
||||
|
||||
requestAbort(controller, undefined, {
|
||||
source: 'query_guard',
|
||||
subsystem: 'query_engine',
|
||||
controllerRole: 'query-root',
|
||||
})
|
||||
|
||||
await expect(decisionPromise).resolves.toMatchObject({
|
||||
behavior: 'ask',
|
||||
})
|
||||
expect(queue).toHaveLength(0)
|
||||
const trace = __getInterruptionTraceSnapshotForTests()
|
||||
const abortRequest = trace.find(entry => entry.event === 'abort.requested')
|
||||
expect(
|
||||
trace.find(entry => entry.event === 'permission.abort_resolved'),
|
||||
).toMatchObject({
|
||||
source: 'query_guard',
|
||||
subsystem: 'in_process_permission_bridge',
|
||||
outcome: 'denied',
|
||||
causalEventId: abortRequest?.eventId,
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,61 @@
|
||||
import {
|
||||
getInterruptionSignalAbortTrace,
|
||||
tracePermissionAbortResolution,
|
||||
} from '../interruptionTrace.js'
|
||||
|
||||
export type InProcessPermissionAbortCompleter = {
|
||||
claim(): boolean
|
||||
completeAbort(
|
||||
source: string | undefined,
|
||||
causalEventId: string | undefined,
|
||||
): boolean
|
||||
completeSignalAbort(): boolean
|
||||
isSettled(): boolean
|
||||
}
|
||||
|
||||
export function createInProcessPermissionAbortCompleter(
|
||||
signal: AbortSignal,
|
||||
onAbortSettled: () => void,
|
||||
): InProcessPermissionAbortCompleter {
|
||||
let settled = false
|
||||
|
||||
const claim = (): boolean => {
|
||||
if (settled) return false
|
||||
settled = true
|
||||
signal.removeEventListener('abort', onSignalAbort)
|
||||
return true
|
||||
}
|
||||
|
||||
const completeAbort = (
|
||||
source: string | undefined,
|
||||
causalEventId: string | undefined,
|
||||
): boolean => {
|
||||
if (!claim()) return false
|
||||
tracePermissionAbortResolution(
|
||||
source,
|
||||
causalEventId,
|
||||
'in_process_permission_bridge',
|
||||
)
|
||||
onAbortSettled()
|
||||
return true
|
||||
}
|
||||
|
||||
const completeSignalAbort = (): boolean => {
|
||||
const trace = getInterruptionSignalAbortTrace(signal)
|
||||
return completeAbort(trace.source, trace.causalEventId)
|
||||
}
|
||||
|
||||
const onSignalAbort = () => {
|
||||
completeSignalAbort()
|
||||
}
|
||||
|
||||
signal.addEventListener('abort', onSignalAbort, { once: true })
|
||||
if (signal.aborted) onSignalAbort()
|
||||
|
||||
return {
|
||||
claim,
|
||||
completeAbort,
|
||||
completeSignalAbort,
|
||||
isSettled: () => settled,
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,9 @@ import {
|
||||
} from '../../tasks/LocalAgentTask/LocalAgentTask.js'
|
||||
import type { CustomAgentDefinition } from '../../tools/AgentTool/loadAgentsDir.js'
|
||||
import { runAgent } from '../../tools/AgentTool/runAgent.js'
|
||||
import {
|
||||
registerInterruptionController,
|
||||
} from '../interruptionTrace.js'
|
||||
import { awaitClassifierAutoApproval } from '../../tools/BashTool/bashPermissions.js'
|
||||
import { BASH_TOOL_NAME } from '../../tools/BashTool/toolName.js'
|
||||
import { SEND_MESSAGE_TOOL_NAME } from '../../tools/SendMessageTool/constants.js'
|
||||
@@ -126,6 +129,7 @@ import {
|
||||
sendPermissionRequestViaMailbox,
|
||||
} from './permissionSync.js'
|
||||
import { TEAMMATE_SYSTEM_PROMPT_ADDENDUM } from './teammatePromptAddendum.js'
|
||||
import { createInProcessPermissionAbortCompleter } from './inProcessPermissionAbort.js'
|
||||
|
||||
type SetAppStateFn = (updater: (prev: AppState) => AppState) => void
|
||||
|
||||
@@ -143,7 +147,7 @@ const PERMISSION_POLL_INTERVAL_MS = 500
|
||||
* sends a permission request to the leader's inbox, waits for the response
|
||||
* in the teammate's own mailbox.
|
||||
*/
|
||||
function createInProcessCanUseTool(
|
||||
export function createInProcessCanUseTool(
|
||||
identity: TeammateIdentity,
|
||||
abortController: AbortController,
|
||||
onPermissionWaitMs?: (waitMs: number) => void,
|
||||
@@ -259,7 +263,6 @@ function createInProcessCanUseTool(
|
||||
// Standard path: use ToolUseConfirm dialog with worker badge
|
||||
if (setToolUseConfirmQueue) {
|
||||
return new Promise<PermissionDecision>(resolve => {
|
||||
let decisionMade = false
|
||||
const permissionStartMs = Date.now()
|
||||
|
||||
// Report permission wait time to the caller so it can be
|
||||
@@ -268,19 +271,20 @@ function createInProcessCanUseTool(
|
||||
onPermissionWaitMs?.(Date.now() - permissionStartMs)
|
||||
}
|
||||
|
||||
const onAbortListener = () => {
|
||||
if (decisionMade) return
|
||||
decisionMade = true
|
||||
const completion = createInProcessPermissionAbortCompleter(
|
||||
abortController.signal,
|
||||
() => {
|
||||
reportPermissionWait()
|
||||
resolve({ behavior: 'ask', message: SUBAGENT_REJECT_MESSAGE })
|
||||
setToolUseConfirmQueue(queue =>
|
||||
queue.filter(item => item.toolUseID !== toolUseID),
|
||||
)
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
abortController.signal.addEventListener('abort', onAbortListener, {
|
||||
once: true,
|
||||
})
|
||||
if (completion.isSettled()) {
|
||||
return
|
||||
}
|
||||
|
||||
setToolUseConfirmQueue(queue => [
|
||||
...queue,
|
||||
@@ -299,15 +303,8 @@ function createInProcessCanUseTool(
|
||||
onUserInteraction() {
|
||||
// No-op for teammates (no classifier auto-approval)
|
||||
},
|
||||
onAbort() {
|
||||
if (decisionMade) return
|
||||
decisionMade = true
|
||||
abortController.signal.removeEventListener(
|
||||
'abort',
|
||||
onAbortListener,
|
||||
)
|
||||
reportPermissionWait()
|
||||
resolve({ behavior: 'ask', message: SUBAGENT_REJECT_MESSAGE })
|
||||
onAbort(source, causalEventId) {
|
||||
completion.completeAbort(source, causalEventId)
|
||||
},
|
||||
async onAllow(
|
||||
updatedInput: Record<string, unknown>,
|
||||
@@ -315,12 +312,7 @@ function createInProcessCanUseTool(
|
||||
feedback?: string,
|
||||
contentBlocks?: ContentBlockParam[],
|
||||
) {
|
||||
if (decisionMade) return
|
||||
decisionMade = true
|
||||
abortController.signal.removeEventListener(
|
||||
'abort',
|
||||
onAbortListener,
|
||||
)
|
||||
if (!completion.claim()) return
|
||||
reportPermissionWait()
|
||||
const approval = await guardExternalApproval(
|
||||
updatedInput,
|
||||
@@ -365,12 +357,7 @@ function createInProcessCanUseTool(
|
||||
})
|
||||
},
|
||||
onReject(feedback?: string, contentBlocks?: ContentBlockParam[]) {
|
||||
if (decisionMade) return
|
||||
decisionMade = true
|
||||
abortController.signal.removeEventListener(
|
||||
'abort',
|
||||
onAbortListener,
|
||||
)
|
||||
if (!completion.claim()) return
|
||||
reportPermissionWait()
|
||||
const message = feedback
|
||||
? `${SUBAGENT_REJECT_MESSAGE_WITH_REASON_PREFIX}${feedback}`
|
||||
@@ -378,7 +365,7 @@ function createInProcessCanUseTool(
|
||||
resolve({ behavior: 'ask', message, contentBlocks })
|
||||
},
|
||||
async recheckPermission() {
|
||||
if (decisionMade) return
|
||||
if (completion.isSettled()) return
|
||||
const freshResult = await hasPermissionsToUseTool(
|
||||
tool,
|
||||
input,
|
||||
@@ -386,12 +373,10 @@ function createInProcessCanUseTool(
|
||||
assistantMessage,
|
||||
toolUseID,
|
||||
)
|
||||
if (freshResult.behavior === 'allow') {
|
||||
decisionMade = true
|
||||
abortController.signal.removeEventListener(
|
||||
'abort',
|
||||
onAbortListener,
|
||||
)
|
||||
if (
|
||||
freshResult.behavior === 'allow' &&
|
||||
completion.claim()
|
||||
) {
|
||||
reportPermissionWait()
|
||||
setToolUseConfirmQueue(queue =>
|
||||
queue.filter(item => item.toolUseID !== toolUseID),
|
||||
@@ -421,6 +406,26 @@ function createInProcessCanUseTool(
|
||||
workerColor: identity.color,
|
||||
teamName: identity.teamName,
|
||||
})
|
||||
let pollInterval: ReturnType<typeof setInterval> | undefined
|
||||
|
||||
function cleanup() {
|
||||
if (pollInterval !== undefined) {
|
||||
clearInterval(pollInterval)
|
||||
}
|
||||
unregisterPermissionCallback(request.id)
|
||||
}
|
||||
|
||||
const completion = createInProcessPermissionAbortCompleter(
|
||||
abortController.signal,
|
||||
() => {
|
||||
cleanup()
|
||||
resolve({ behavior: 'ask', message: SUBAGENT_REJECT_MESSAGE })
|
||||
},
|
||||
)
|
||||
|
||||
if (completion.isSettled()) {
|
||||
return
|
||||
}
|
||||
|
||||
// Register callback to be invoked when the leader responds
|
||||
registerPermissionCallback({
|
||||
@@ -432,6 +437,7 @@ function createInProcessCanUseTool(
|
||||
_feedback?: string,
|
||||
contentBlocks?: ContentBlockParam[],
|
||||
) {
|
||||
if (!completion.claim()) return
|
||||
cleanup()
|
||||
const finalInput =
|
||||
updatedInput && Object.keys(updatedInput).length > 0
|
||||
@@ -460,6 +466,7 @@ function createInProcessCanUseTool(
|
||||
})
|
||||
},
|
||||
onReject(feedback?: string, contentBlocks?: ContentBlockParam[]) {
|
||||
if (!completion.claim()) return
|
||||
cleanup()
|
||||
const message = feedback
|
||||
? `${SUBAGENT_REJECT_MESSAGE_WITH_REASON_PREFIX}${feedback}`
|
||||
@@ -472,11 +479,10 @@ function createInProcessCanUseTool(
|
||||
void sendPermissionRequestViaMailbox(request)
|
||||
|
||||
// Poll teammate's mailbox for the response
|
||||
const pollInterval = setInterval(
|
||||
async (abortController, cleanup, resolve, identity, request) => {
|
||||
pollInterval = setInterval(
|
||||
async (completion, identity, request) => {
|
||||
if (abortController.signal.aborted) {
|
||||
cleanup()
|
||||
resolve({ behavior: 'ask', message: SUBAGENT_REJECT_MESSAGE })
|
||||
completion.completeSignalAbort()
|
||||
return
|
||||
}
|
||||
|
||||
@@ -514,27 +520,10 @@ function createInProcessCanUseTool(
|
||||
}
|
||||
},
|
||||
PERMISSION_POLL_INTERVAL_MS,
|
||||
abortController,
|
||||
cleanup,
|
||||
resolve,
|
||||
completion,
|
||||
identity,
|
||||
request,
|
||||
)
|
||||
|
||||
const onAbortListener = () => {
|
||||
cleanup()
|
||||
resolve({ behavior: 'ask', message: SUBAGENT_REJECT_MESSAGE })
|
||||
}
|
||||
|
||||
abortController.signal.addEventListener('abort', onAbortListener, {
|
||||
once: true,
|
||||
})
|
||||
|
||||
function cleanup() {
|
||||
clearInterval(pollInterval)
|
||||
unregisterPermissionCallback(request.id)
|
||||
abortController.signal.removeEventListener('abort', onAbortListener)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1168,6 +1157,12 @@ export async function runInProcessTeammate(
|
||||
// This allows Escape to stop current work without killing the whole teammate.
|
||||
// The lifecycle abortController still kills the whole teammate if needed.
|
||||
const currentWorkAbortController = createAbortController()
|
||||
registerInterruptionController(currentWorkAbortController, {
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-turn',
|
||||
subagentId: identity.agentId,
|
||||
querySource: 'agent:custom',
|
||||
})
|
||||
|
||||
// Store the work controller in task state so UI can abort it
|
||||
updateTaskState(
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import { afterEach, beforeEach, expect, test } from 'bun:test'
|
||||
import type { AppState } from '../../state/AppState.js'
|
||||
import { getDefaultAppState } from '../../state/AppStateStore.js'
|
||||
import type { InProcessTeammateTaskState } from '../../tasks/InProcessTeammateTask/types.js'
|
||||
import {
|
||||
__getInterruptionTraceSnapshotForTests,
|
||||
__resetInterruptionTraceForTests,
|
||||
} from '../interruptionTrace.js'
|
||||
import {
|
||||
acquireSharedMutationLock,
|
||||
releaseSharedMutationLock,
|
||||
} from '../../test/sharedMutationLock.js'
|
||||
import { killInProcessTeammate } from './spawnInProcess.js'
|
||||
|
||||
const originalTrace = process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
|
||||
beforeEach(async () => {
|
||||
await acquireSharedMutationLock(
|
||||
'utils/swarm/spawnInProcess.interruptionTrace.test.ts',
|
||||
)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
try {
|
||||
__resetInterruptionTraceForTests()
|
||||
if (originalTrace === undefined) {
|
||||
delete process.env.OPENCLAUDE_INTERRUPT_TRACE
|
||||
} else {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = originalTrace
|
||||
}
|
||||
} finally {
|
||||
releaseSharedMutationLock()
|
||||
}
|
||||
})
|
||||
|
||||
test('records the teammate lifecycle source before a kill abort', () => {
|
||||
process.env.OPENCLAUDE_INTERRUPT_TRACE = '1'
|
||||
const abortController = new AbortController()
|
||||
const taskId = 'teammate-task-1'
|
||||
const task = {
|
||||
id: taskId,
|
||||
type: 'in_process_teammate',
|
||||
status: 'running',
|
||||
description: 'test teammate',
|
||||
startTime: Date.now(),
|
||||
outputFile: '/tmp/test-teammate-output',
|
||||
outputOffset: 0,
|
||||
notified: false,
|
||||
identity: {
|
||||
agentId: 'researcher',
|
||||
agentName: 'researcher',
|
||||
teamName: '',
|
||||
planModeRequired: false,
|
||||
parentSessionId: 'parent-session',
|
||||
},
|
||||
prompt: 'test',
|
||||
abortController,
|
||||
awaitingPlanApproval: false,
|
||||
permissionMode: 'default',
|
||||
isIdle: false,
|
||||
shutdownRequested: false,
|
||||
pendingUserMessages: [],
|
||||
lastReportedToolCount: 0,
|
||||
lastReportedTokenCount: 0,
|
||||
} satisfies InProcessTeammateTaskState
|
||||
let state: AppState = {
|
||||
...getDefaultAppState(),
|
||||
tasks: { [taskId]: task },
|
||||
}
|
||||
|
||||
expect(
|
||||
killInProcessTeammate(taskId, updater => {
|
||||
state = updater(state)
|
||||
}),
|
||||
).toBe(true)
|
||||
|
||||
expect(abortController.signal.aborted).toBe(true)
|
||||
expect(
|
||||
__getInterruptionTraceSnapshotForTests().find(
|
||||
entry => entry.event === 'abort.requested',
|
||||
),
|
||||
).toMatchObject({
|
||||
source: 'task_stop',
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: 'researcher',
|
||||
})
|
||||
})
|
||||
@@ -27,6 +27,10 @@ import { createAbortController } from '../abortController.js'
|
||||
import { formatAgentId } from '../agentId.js'
|
||||
import { registerCleanup } from '../cleanupRegistry.js'
|
||||
import { logForDebugging } from '../debug.js'
|
||||
import {
|
||||
registerInterruptionController,
|
||||
requestAbort,
|
||||
} from '../interruptionTrace.js'
|
||||
import { emitTaskTerminatedSdk } from '../sdkEventQueue.js'
|
||||
import { evictTaskOutput } from '../task/diskOutput.js'
|
||||
import {
|
||||
@@ -44,6 +48,11 @@ import { removeMemberByAgentId } from './teamHelpers.js'
|
||||
|
||||
type SetAppStateFn = (updater: (prev: AppState) => AppState) => void
|
||||
|
||||
export type InProcessTeammateKillTrace = {
|
||||
source: string
|
||||
causalEventId?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal context required for spawning an in-process teammate.
|
||||
* This is a subset of ToolUseContext - only what spawnInProcessTeammate actually uses.
|
||||
@@ -120,6 +129,11 @@ export async function spawnInProcessTeammate(
|
||||
// Create independent AbortController for this teammate
|
||||
// Teammates should not be aborted when the leader's query is interrupted
|
||||
const abortController = createAbortController()
|
||||
registerInterruptionController(abortController, {
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: agentId,
|
||||
})
|
||||
|
||||
// Get parent session ID for transcript correlation
|
||||
const parentSessionId = getSessionId()
|
||||
@@ -182,7 +196,12 @@ export async function spawnInProcessTeammate(
|
||||
// Register cleanup handler for graceful shutdown
|
||||
const unregisterCleanup = registerCleanup(async () => {
|
||||
logForDebugging(`[spawnInProcessTeammate] Cleanup called for ${agentId}`)
|
||||
abortController.abort()
|
||||
requestAbort(abortController, undefined, {
|
||||
source: 'graceful_shutdown',
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: agentId,
|
||||
})
|
||||
// Task state will be updated by the execution loop when it detects abort
|
||||
})
|
||||
taskState.unregisterCleanup = unregisterCleanup
|
||||
@@ -227,6 +246,7 @@ export async function spawnInProcessTeammate(
|
||||
export function killInProcessTeammate(
|
||||
taskId: string,
|
||||
setAppState: SetAppStateFn,
|
||||
trace: InProcessTeammateKillTrace = { source: 'task_stop' },
|
||||
): boolean {
|
||||
let killed = false
|
||||
let teamName: string | null = null
|
||||
@@ -253,7 +273,15 @@ export function killInProcessTeammate(
|
||||
description = teammateTask.description
|
||||
|
||||
// Abort the controller to stop execution
|
||||
teammateTask.abortController?.abort()
|
||||
if (teammateTask.abortController) {
|
||||
requestAbort(teammateTask.abortController, undefined, {
|
||||
source: trace.source,
|
||||
subsystem: 'in_process_teammate',
|
||||
controllerRole: 'subagent-lifecycle',
|
||||
subagentId: teammateTask.identity.agentId,
|
||||
causalEventId: trace.causalEventId,
|
||||
})
|
||||
}
|
||||
|
||||
// Call cleanup handler
|
||||
teammateTask.unregisterCleanup?.()
|
||||
|
||||
@@ -128,6 +128,21 @@ describe('Engine lazy-init guard (COR-1)', () => {
|
||||
expect(() => q.close()).not.toThrow()
|
||||
})
|
||||
|
||||
test('QueryImpl close() aborts its wrapper controller after an engine override', () => {
|
||||
const abortController = new AbortController()
|
||||
const q = query({
|
||||
prompt: 'test',
|
||||
options: { cwd: process.cwd(), abortController },
|
||||
})
|
||||
;(q as unknown as {
|
||||
setEngine(engine: { interrupt(): void }): void
|
||||
}).setEngine({ interrupt() {} })
|
||||
|
||||
q.close()
|
||||
|
||||
expect(abortController.signal.aborted).toBe(true)
|
||||
})
|
||||
|
||||
test('SDKSession getMessages() works after construction', async () => {
|
||||
const { unstable_v2_createSession } = await import('../../src/entrypoints/sdk/index.js')
|
||||
const session = unstable_v2_createSession({
|
||||
|
||||
@@ -151,6 +151,19 @@ describe('V2: session interrupt', () => {
|
||||
expect(() => session.interrupt()).not.toThrow()
|
||||
})
|
||||
|
||||
test('session.close() aborts its wrapper controller after an engine override', () => {
|
||||
const abortController = new AbortController()
|
||||
const session = unstable_v2_createSession({
|
||||
cwd: process.cwd(),
|
||||
abortController,
|
||||
})
|
||||
attachMockEngine(session, new MockQueryEngine())
|
||||
|
||||
session.close()
|
||||
|
||||
expect(abortController.signal.aborted).toBe(true)
|
||||
})
|
||||
|
||||
test('session with external abortController — abort signal propagates', async () => {
|
||||
const ac = new AbortController()
|
||||
const session = unstable_v2_createSession({
|
||||
|
||||
Reference in New Issue
Block a user