mirror of
https://github.com/Gitlawb/openclaude.git
synced 2026-08-24 02:34:15 -05:00
perf(tools): preserve UTF-8-safe head and tail in persisted previews (#1960)
* perf(tools): preserve UTF-8-safe head and tail in persisted previews * fix(tools): derive persisted preview size from file * fix(tools): report exact persisted preview bytes * fix(tools): bound and sanitize persisted previews * fix(tools): reconcile sanitized preview metadata
This commit is contained in:
@@ -77,6 +77,7 @@ import {
|
||||
getBinaryBlobSavedMessage,
|
||||
getFormatDescription,
|
||||
getLargeOutputInstructions,
|
||||
getLargeOutputPersistenceFailureInstructions,
|
||||
persistBinaryContent,
|
||||
} from '../../utils/mcpOutputStorage.js'
|
||||
import {
|
||||
@@ -2857,20 +2858,22 @@ export async function processMCPResult(
|
||||
|
||||
if (isPersistError(persistResult)) {
|
||||
// If file save failed, fall back to returning truncated content info
|
||||
const contentLength = contentStr.length
|
||||
logEvent('tengu_mcp_large_result_handled', {
|
||||
outcome: 'truncated',
|
||||
reason: 'persist_failed',
|
||||
sizeEstimateTokens,
|
||||
} as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS)
|
||||
return `Error: result (${contentLength.toLocaleString()} characters) exceeds maximum allowed tokens. Failed to save output to file: ${persistResult.error}. If this MCP server provides pagination or filtering tools, use them to retrieve specific portions of the data.`
|
||||
return getLargeOutputPersistenceFailureInstructions(
|
||||
contentStr,
|
||||
persistResult.error,
|
||||
)
|
||||
}
|
||||
|
||||
logEvent('tengu_mcp_large_result_handled', {
|
||||
outcome: 'persisted',
|
||||
reason: 'file_saved',
|
||||
sizeEstimateTokens,
|
||||
persistedSizeChars: persistResult.originalSize,
|
||||
persistedSizeBytes: persistResult.originalSize,
|
||||
} as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS)
|
||||
|
||||
const formatDescription = getFormatDescription(type, schema)
|
||||
|
||||
@@ -18,6 +18,10 @@ import {
|
||||
import { getEmptyToolPermissionContext } from '../../Tool.js'
|
||||
import { ShellError } from '../../utils/errors.js'
|
||||
import { formatError } from '../../utils/toolErrors.js'
|
||||
import {
|
||||
generatePreview,
|
||||
PREVIEW_SIZE_BYTES,
|
||||
} from '../../utils/toolResultStorage.js'
|
||||
|
||||
// Regression for #1231 — non-zero exit must not hide captured stdout/stderr.
|
||||
// The Bash tool runs with a merged-fd setup (both streams to one file), so
|
||||
@@ -51,6 +55,63 @@ async function expectShellError(command: string): Promise<ShellError> {
|
||||
}
|
||||
|
||||
describe('BashTool error output (#1231)', () => {
|
||||
test('uses the persisted file preview for the model-facing success result', () => {
|
||||
const fullOutput = `COMMAND CONTEXT\n${'routine output\n'.repeat(300)}FAILURE ROOT\n`
|
||||
const preview = generatePreview(fullOutput, PREVIEW_SIZE_BYTES).preview
|
||||
const mapped = BashTool.mapToolResultToToolResultBlockParam(
|
||||
{
|
||||
stdout: 'captured head only',
|
||||
stderr: '',
|
||||
interrupted: false,
|
||||
persistedOutputPath: '/tmp/full-output.txt',
|
||||
persistedOutputSize: 42_100,
|
||||
persistedOutputPreview: preview,
|
||||
persistedOutputPreviewStrategy: 'head-tail',
|
||||
} as never,
|
||||
'toolu_persisted_preview',
|
||||
)
|
||||
|
||||
expect(String(mapped.content)).toContain(preview)
|
||||
expect(String(mapped.content)).toContain('UTF-8-safe head and tail')
|
||||
expect(String(mapped.content)).not.toContain('complete available inline output')
|
||||
expect(Buffer.byteLength(preview, 'utf8')).toBeLessThanOrEqual(
|
||||
PREVIEW_SIZE_BYTES,
|
||||
)
|
||||
})
|
||||
|
||||
test('labels a small captured-only fallback as partial, not complete', () => {
|
||||
const mapped = BashTool.mapToolResultToToolResultBlockParam(
|
||||
{
|
||||
stdout: 'small captured head',
|
||||
stderr: '',
|
||||
interrupted: false,
|
||||
persistedOutputPath: '/tmp/full-output.txt',
|
||||
persistedOutputSize: 42_100,
|
||||
} as never,
|
||||
'toolu_captured_fallback',
|
||||
)
|
||||
|
||||
expect(String(mapped.content)).toContain('UTF-8-safe head-only partial output')
|
||||
expect(String(mapped.content)).not.toContain('complete available inline output')
|
||||
})
|
||||
|
||||
test('under-claims a supplied preview when its strategy is missing', () => {
|
||||
const mapped = BashTool.mapToolResultToToolResultBlockParam(
|
||||
{
|
||||
stdout: 'captured head only',
|
||||
stderr: '',
|
||||
interrupted: false,
|
||||
persistedOutputPath: '/tmp/full-output.txt',
|
||||
persistedOutputSize: 42_100,
|
||||
persistedOutputPreview: 'preview with unknown provenance',
|
||||
} as never,
|
||||
'toolu_preview_without_strategy',
|
||||
)
|
||||
|
||||
expect(String(mapped.content)).toContain('UTF-8-safe head-only partial output')
|
||||
expect(String(mapped.content)).not.toContain('UTF-8-safe head and tail')
|
||||
})
|
||||
|
||||
test('captured stdout/stderr appear in formatted error on non-zero exit', async () => {
|
||||
const err = await expectShellError(
|
||||
'echo stdout-line; echo stderr-line >&2; exit 1',
|
||||
@@ -70,6 +131,18 @@ describe('BashTool error output (#1231)', () => {
|
||||
expect(formatted.toLowerCase()).toContain('not found')
|
||||
})
|
||||
|
||||
test('strips Claude Code hints from non-zero output when no persisted preview is available', async () => {
|
||||
const hint =
|
||||
'<claude-code-hint v="1" type="plugin" value="example@claude-plugins-official" />'
|
||||
const err = await expectShellError(
|
||||
`printf '%s\\n' '${hint}'; printf 'FAILURE ROOT\\n'; exit 1`,
|
||||
)
|
||||
const formatted = formatError(err)
|
||||
|
||||
expect(formatted).toContain('FAILURE ROOT')
|
||||
expect(formatted).not.toContain('<claude-code-hint')
|
||||
})
|
||||
|
||||
test('captured output is carried on the stdout slot (semantic mapping)', async () => {
|
||||
const err = await expectShellError('echo merged-line; exit 2')
|
||||
expect(err.stdout).toContain('merged-line')
|
||||
@@ -150,7 +223,7 @@ describe('BashTool error output (#1231)', () => {
|
||||
let persistedPath: string | undefined
|
||||
try {
|
||||
const err = await expectShellError(
|
||||
`for i in $(seq 1 700); do printf 'line %04d %s\\n' "$i" "padding-to-make-this-line-fat-enough-to-cross-the-limit"; done; exit 1`,
|
||||
`for i in $(seq 1 700); do printf 'line %04d %s\\n' "$i" "padding-to-make-this-line-fat-enough-to-cross-the-limit"; done; printf 'FAILURE ROOT: src/index.ts:42\\n'; exit 1`,
|
||||
)
|
||||
expect(err.code).toBe(1)
|
||||
const formatted = formatError(err)
|
||||
@@ -165,6 +238,9 @@ describe('BashTool error output (#1231)', () => {
|
||||
expect(match).not.toBeNull()
|
||||
persistedPath = match?.[1]
|
||||
expect(persistedPath).toBeDefined()
|
||||
expect(formatted).toContain('FAILURE ROOT: src/index.ts:42')
|
||||
expect(formatted).toMatch(/… \d+ bytes omitted …/)
|
||||
expect(formatted).not.toContain('line 0200')
|
||||
|
||||
// The saved file must actually be readable and contain the late output
|
||||
// that #1359 needs the model to recover — i.e. the tail line, which the
|
||||
@@ -194,11 +270,62 @@ describe('BashTool error output (#1231)', () => {
|
||||
expect(hint).toContain('/tmp/out')
|
||||
})
|
||||
|
||||
test('capped hint distinguishes preview tail bytes from saved bytes', () => {
|
||||
const hint = appendPersistedOutputHint(
|
||||
'captured output',
|
||||
'/tmp/out',
|
||||
MAX_PERSISTED_SHELL_OUTPUT_SIZE + 4096,
|
||||
true,
|
||||
'COMMAND CONTEXT\n… 4096 bytes omitted …\nFAILURE ROOT',
|
||||
'head-tail',
|
||||
)
|
||||
|
||||
expect(hint).toContain('preview may include tail bytes not saved at that path')
|
||||
})
|
||||
|
||||
test('hint keeps "full output" wording when the roll file fit under the cap', () => {
|
||||
const hint = appendPersistedOutputHint('preview', '/tmp/out', 1234, false)
|
||||
expect(hint).toMatch(/full output \(1234 bytes\) saved to \/tmp\/out; read with the Read tool/)
|
||||
})
|
||||
|
||||
test('bounded error preview replaces the captured head but preserves sandbox diagnostics', () => {
|
||||
const captured = `${'captured duplicate\n'.repeat(2_000)}<sandbox_violations>${'literal command output'.repeat(2_000)}</sandbox_violations>`
|
||||
const preview = 'COMMAND CONTEXT\n… 42,000 bytes omitted …\nFAILURE ROOT'
|
||||
const sandboxDiagnostics =
|
||||
'<sandbox_violations>actual denied write</sandbox_violations>'
|
||||
const hint = appendPersistedOutputHint(
|
||||
captured,
|
||||
'/tmp/out',
|
||||
42_100,
|
||||
false,
|
||||
preview,
|
||||
'head-tail',
|
||||
sandboxDiagnostics,
|
||||
)
|
||||
|
||||
expect(hint).toContain(preview)
|
||||
expect(hint).not.toContain('captured duplicate')
|
||||
expect(hint).not.toContain('literal command output')
|
||||
expect(hint).toContain(
|
||||
'<sandbox_violations>actual denied write</sandbox_violations>',
|
||||
)
|
||||
expect(Buffer.byteLength(hint, 'utf8')).toBeLessThan(3_000)
|
||||
})
|
||||
|
||||
test('labels a head-only persisted preview honestly', () => {
|
||||
const hint = appendPersistedOutputHint(
|
||||
'captured output',
|
||||
'/tmp/out',
|
||||
42_100,
|
||||
false,
|
||||
'COMMAND CONTEXT',
|
||||
'head-only',
|
||||
)
|
||||
|
||||
expect(hint).toContain('UTF-8-safe head-only partial')
|
||||
expect(hint).not.toContain('UTF-8-safe head and tail')
|
||||
})
|
||||
|
||||
// Follow-up to #1359 — when the roll file exceeds the cap, the cap must be
|
||||
// applied to the saved copy, NOT to the shell's rolled-output source. The
|
||||
// error fallback and resizeShellImageOutput still read the source, so
|
||||
@@ -246,6 +373,12 @@ describe('BashTool error output (#1231)', () => {
|
||||
expect(persisted).not.toBeNull()
|
||||
dest = persisted!.path
|
||||
expect(persisted!.truncated).toBe(true)
|
||||
expect(persisted!.preview).toStartWith('A')
|
||||
expect(persisted!.preview).toEndWith('B'.repeat(790))
|
||||
expect(persisted!.preview).toMatch(/… \d+ bytes omitted …/)
|
||||
expect(Buffer.byteLength(persisted!.preview!, 'utf8')).toBeLessThanOrEqual(
|
||||
PREVIEW_SIZE_BYTES,
|
||||
)
|
||||
const saved = readFileSync(dest, 'utf8')
|
||||
expect(saved.length).toBe(cap)
|
||||
expect(saved).toBe(head) // exactly the head, no tail byte leaked in
|
||||
@@ -255,4 +388,125 @@ describe('BashTool error output (#1231)', () => {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('strips and reports a retained-tail Claude Code hint without changing the saved file', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'bash-persist-hint-'))
|
||||
const source = join(dir, 'roll.txt')
|
||||
const hint =
|
||||
'<claude-code-hint v="1" type="plugin" value="example@claude-plugins-official" />'
|
||||
const body = `COMMAND CONTEXT\n${'routine output\n'.repeat(300)}${hint}\nFAILURE ROOT\n`
|
||||
writeFileSync(source, body)
|
||||
let dest: string | undefined
|
||||
|
||||
try {
|
||||
const persisted = await persistShellOutputFile(
|
||||
source,
|
||||
'persist-hint-test',
|
||||
MAX_PERSISTED_SHELL_OUTPUT_SIZE,
|
||||
'example-cli run',
|
||||
)
|
||||
expect(persisted).not.toBeNull()
|
||||
dest = persisted!.path
|
||||
expect(persisted!.preview).toContain('FAILURE ROOT')
|
||||
expect(persisted!.preview).not.toContain('<claude-code-hint')
|
||||
const marker = persisted!.preview!.match(/… (\d+) bytes omitted …/)
|
||||
expect(marker).toBeDefined()
|
||||
const displayedOutput = persisted!.preview!.replace(marker![0], '')
|
||||
expect(Number(marker![1])).toBe(
|
||||
Buffer.byteLength(body, 'utf8') -
|
||||
Buffer.byteLength(displayedOutput, 'utf8'),
|
||||
)
|
||||
expect(persisted!.previewHints).toEqual([
|
||||
{
|
||||
v: 1,
|
||||
type: 'plugin',
|
||||
value: 'example@claude-plugins-official',
|
||||
sourceCommand: 'example-cli',
|
||||
},
|
||||
])
|
||||
expect(readFileSync(dest, 'utf8')).toBe(body)
|
||||
} finally {
|
||||
if (dest && existsSync(dest)) rmSync(dest, { force: true })
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('downgrades a complete preview when a hint line is removed', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'bash-persist-complete-hint-'))
|
||||
const source = join(dir, 'roll.txt')
|
||||
const hint =
|
||||
'<claude-code-hint v="1" type="plugin" value="example@claude-plugins-official" />'
|
||||
const body = `COMMAND CONTEXT\n${hint}\nFAILURE ROOT\n`
|
||||
writeFileSync(source, body)
|
||||
let dest: string | undefined
|
||||
|
||||
try {
|
||||
const persisted = await persistShellOutputFile(
|
||||
source,
|
||||
'persist-complete-hint-test',
|
||||
MAX_PERSISTED_SHELL_OUTPUT_SIZE,
|
||||
'example-cli run',
|
||||
)
|
||||
expect(persisted).not.toBeNull()
|
||||
dest = persisted!.path
|
||||
expect(persisted!.previewStrategy).toBe('head-only')
|
||||
expect(persisted!.preview).not.toContain('<claude-code-hint')
|
||||
|
||||
const mapped = BashTool.mapToolResultToToolResultBlockParam(
|
||||
{
|
||||
stdout: 'captured head only',
|
||||
stderr: '',
|
||||
interrupted: false,
|
||||
persistedOutputPath: persisted!.path,
|
||||
persistedOutputSize: persisted!.size,
|
||||
persistedOutputPreview: persisted!.preview,
|
||||
persistedOutputPreviewStrategy: persisted!.previewStrategy,
|
||||
} as never,
|
||||
'toolu_sanitized_complete_preview',
|
||||
)
|
||||
expect(String(mapped.content)).toContain(
|
||||
'UTF-8-safe head-only partial output',
|
||||
)
|
||||
expect(String(mapped.content)).not.toContain(
|
||||
'complete available inline output',
|
||||
)
|
||||
expect(readFileSync(dest, 'utf8')).toBe(body)
|
||||
} finally {
|
||||
if (dest && existsSync(dest)) rmSync(dest, { force: true })
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('falls back to head-only when a retained hint contains malformed UTF-8', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'bash-persist-malformed-hint-'))
|
||||
const source = join(dir, 'roll.txt')
|
||||
const body = Buffer.concat([
|
||||
Buffer.from(`COMMAND CONTEXT\n${'routine output\n'.repeat(300)}`),
|
||||
Buffer.from('<claude-code-hint v="1" type="plugin" value="example'),
|
||||
Buffer.from([0xff]),
|
||||
Buffer.from('@claude-plugins-official" />\nFAILURE ROOT\n'),
|
||||
])
|
||||
writeFileSync(source, body)
|
||||
let dest: string | undefined
|
||||
|
||||
try {
|
||||
const persisted = await persistShellOutputFile(
|
||||
source,
|
||||
'persist-malformed-hint-test',
|
||||
MAX_PERSISTED_SHELL_OUTPUT_SIZE,
|
||||
'example-cli run',
|
||||
)
|
||||
expect(persisted).not.toBeNull()
|
||||
dest = persisted!.path
|
||||
expect(persisted!.previewStrategy).toBe('head-only')
|
||||
expect(persisted!.preview).toContain('COMMAND CONTEXT')
|
||||
expect(persisted!.preview).not.toContain('FAILURE ROOT')
|
||||
expect(persisted!.preview).not.toContain('<claude-code-hint')
|
||||
expect(persisted!.preview).not.toMatch(/… \d+ bytes omitted …/)
|
||||
expect(readFileSync(dest)).toEqual(body)
|
||||
} finally {
|
||||
if (dest && existsSync(dest)) rmSync(dest, { force: true })
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
+118
-13
@@ -18,15 +18,16 @@ import type { AgentId } from '../../types/ids.js';
|
||||
import type { AssistantMessage } from '../../types/message.js';
|
||||
import { parseForSecurity } from '../../utils/bash/ast.js';
|
||||
import { splitCommand_DEPRECATED, splitCommandWithOperators } from '../../utils/bash/commands.js';
|
||||
import { extractClaudeCodeHints } from '../../utils/claudeCodeHints.js';
|
||||
import { extractClaudeCodeHints, extractClaudeCodeHintsFromPreview, type ClaudeCodeHint } from '../../utils/claudeCodeHints.js';
|
||||
import { detectCodeIndexingFromCommand } from '../../utils/codeIndexing.js';
|
||||
import { isEnvTruthy } from '../../utils/envUtils.js';
|
||||
import { isENOENT, ShellError } from '../../utils/errors.js';
|
||||
import { isENOENT, ShellError, toError } from '../../utils/errors.js';
|
||||
import { detectFileEncoding, detectLineEndings, getFileModificationTime, writeTextContent } from '../../utils/file.js';
|
||||
import { fileHistoryEnabled, fileHistoryTrackEdit } from '../../utils/fileHistory.js';
|
||||
import { truncate } from '../../utils/format.js';
|
||||
import { getFsImplementation } from '../../utils/fsOperations.js';
|
||||
import { lazySchema } from '../../utils/lazySchema.js';
|
||||
import { logError } from '../../utils/log.js';
|
||||
import { expandPath } from '../../utils/path.js';
|
||||
import type { PermissionResult } from '../../utils/permissions/PermissionResult.js';
|
||||
import { maybeRecordPluginHint } from '../../utils/plugins/hintRecommendation.js';
|
||||
@@ -39,7 +40,7 @@ import { EndTruncatingAccumulator } from '../../utils/stringUtils.js';
|
||||
import { getTaskOutputPath } from '../../utils/task/diskOutput.js';
|
||||
import { TaskOutput } from '../../utils/task/TaskOutput.js';
|
||||
import { isOutputLineTruncated } from '../../utils/terminal.js';
|
||||
import { buildLargeToolResultMessage, ensureToolResultsDir, generatePreview, getToolResultPath, PREVIEW_SIZE_BYTES } from '../../utils/toolResultStorage.js';
|
||||
import { buildLargeToolResultMessage, ensureToolResultsDir, generateFilePreview, generatePreview, getToolResultPath, PREVIEW_SIZE_BYTES, type PreviewStrategy } from '../../utils/toolResultStorage.js';
|
||||
import { userFacingName as fileEditUserFacingName } from '../FileEditTool/UI.js';
|
||||
import { trackGitOperations } from '../shared/gitOperationTracking.js';
|
||||
import { bashToolHasPermission, commandHasAnyCd, matchWildcardPattern, permissionRuleExtractPrefix } from './bashPermissions.js';
|
||||
@@ -304,6 +305,8 @@ const outputSchema = lazySchema(() => z.object({
|
||||
structuredContent: z.array(z.any()).optional().describe('Structured content blocks'),
|
||||
persistedOutputPath: z.string().optional().describe('Path to the persisted full output in tool-results dir (set when output is too large for inline)'),
|
||||
persistedOutputSize: z.number().optional().describe('Total size of the output in bytes (set when output is too large for inline)'),
|
||||
persistedOutputPreview: z.string().optional().describe('UTF-8-safe head/tail preview read from the complete rolled-output source'),
|
||||
persistedOutputPreviewStrategy: z.enum(['complete', 'head-tail', 'head-only']).optional().describe('How the persisted output preview was selected'),
|
||||
persistedOutputTruncated: z.boolean().optional().describe('Whether the persisted file is capped (only the first portion of the output was saved)')
|
||||
}));
|
||||
type OutputSchema = ReturnType<typeof outputSchema>;
|
||||
@@ -324,6 +327,7 @@ import type { BashProgress } from '../../types/tools.js';
|
||||
* command does not blow up disk usage in the user's home dir.
|
||||
*/
|
||||
export const MAX_PERSISTED_SHELL_OUTPUT_SIZE = 64 * 1024 * 1024;
|
||||
const MAX_SANDBOX_DIAGNOSTIC_PREVIEW_BYTES = 512;
|
||||
|
||||
/**
|
||||
* Copy the shell's rolled-output file into the tool-results dir so the model
|
||||
@@ -341,7 +345,8 @@ export async function persistShellOutputFile(
|
||||
sourcePath: string,
|
||||
taskId: string,
|
||||
maxSize: number = MAX_PERSISTED_SHELL_OUTPUT_SIZE,
|
||||
): Promise<{ path: string; size: number; truncated: boolean } | null> {
|
||||
command: string = '',
|
||||
): Promise<{ path: string; size: number; truncated: boolean; preview?: string; previewStrategy?: PreviewStrategy; previewHints?: ClaudeCodeHint[] } | null> {
|
||||
try {
|
||||
const fileStat = await fsStat(sourcePath);
|
||||
const size = fileStat.size;
|
||||
@@ -379,7 +384,25 @@ export async function persistShellOutputFile(
|
||||
// reports as the output total. `truncated` tells the error path that the
|
||||
// saved file is capped at MAX_PERSISTED_SHELL_OUTPUT_SIZE so it does not
|
||||
// describe a partial file as the full output.
|
||||
return { path: dest, size, truncated };
|
||||
const previewSourcePath = truncated ? sourcePath : dest;
|
||||
const previewResult = await generateFilePreview(
|
||||
previewSourcePath,
|
||||
PREVIEW_SIZE_BYTES,
|
||||
).catch(error => {
|
||||
logError(toError(error));
|
||||
return undefined;
|
||||
});
|
||||
const previewExtraction = previewResult
|
||||
? extractClaudeCodeHintsFromPreview(previewResult, command)
|
||||
: undefined;
|
||||
return {
|
||||
path: dest,
|
||||
size,
|
||||
truncated,
|
||||
preview: previewExtraction?.previewResult.preview,
|
||||
previewStrategy: previewExtraction?.previewResult.strategy,
|
||||
previewHints: previewExtraction?.hints,
|
||||
};
|
||||
} catch {
|
||||
// File may already be gone — caller's stdout preview is sufficient.
|
||||
return null;
|
||||
@@ -401,13 +424,36 @@ export function appendPersistedOutputHint(
|
||||
persistedPath: string,
|
||||
persistedSize: number,
|
||||
truncated: boolean,
|
||||
preview?: string,
|
||||
previewStrategy?: PreviewStrategy,
|
||||
sandboxDiagnostics?: string,
|
||||
): string {
|
||||
const capDetail = previewStrategy === 'head-tail'
|
||||
? 'capped; preview may include tail bytes not saved at that path'
|
||||
: 'capped, tail not saved';
|
||||
const hint = truncated
|
||||
? `[output truncated above — first ${MAX_PERSISTED_SHELL_OUTPUT_SIZE} bytes of the ${persistedSize}-byte output saved to ${persistedPath} (capped, tail not saved); read with the Read tool]`
|
||||
? `[output truncated above — first ${MAX_PERSISTED_SHELL_OUTPUT_SIZE} bytes of the ${persistedSize}-byte output saved to ${persistedPath} (${capDetail}); read with the Read tool]`
|
||||
: `[output truncated above — full output (${persistedSize} bytes) saved to ${persistedPath}; read with the Read tool]`;
|
||||
if (!stdout) return hint;
|
||||
const trimmed = stdout.endsWith('\n') ? stdout.slice(0, -1) : stdout;
|
||||
return `${trimmed}\n\n${hint}`;
|
||||
const previewStrategyLabel = previewStrategy === 'head-tail'
|
||||
? 'head and tail'
|
||||
: previewStrategy === 'complete'
|
||||
? 'complete'
|
||||
: 'head-only partial';
|
||||
const previewBlock = preview
|
||||
? `Persisted output preview (UTF-8-safe ${previewStrategyLabel}, ${PREVIEW_SIZE_BYTES}-byte budget):\n${preview}`
|
||||
: '';
|
||||
const boundedSandboxDiagnostics = preview && sandboxDiagnostics
|
||||
? generatePreview(
|
||||
sandboxDiagnostics,
|
||||
MAX_SANDBOX_DIAGNOSTIC_PREVIEW_BYTES,
|
||||
'text',
|
||||
).preview
|
||||
: '';
|
||||
const capturedFallback = preview
|
||||
? ''
|
||||
: stdout.endsWith('\n') ? stdout.slice(0, -1) : stdout;
|
||||
const parts = [capturedFallback, previewBlock, boundedSandboxDiagnostics, hint].filter(Boolean);
|
||||
return parts.join('\n\n');
|
||||
}
|
||||
|
||||
function isAutobackgroundingAllowed(command: string): boolean {
|
||||
@@ -671,6 +717,8 @@ export const BashTool = buildTool({
|
||||
structuredContent,
|
||||
persistedOutputPath,
|
||||
persistedOutputSize,
|
||||
persistedOutputPreview,
|
||||
persistedOutputPreviewStrategy,
|
||||
persistedOutputTruncated
|
||||
}, toolUseID): ToolResultBlockParam {
|
||||
// Handle structured content
|
||||
@@ -700,13 +748,21 @@ export const BashTool = buildTool({
|
||||
// For large output that was persisted to disk, build <persisted-output>
|
||||
// message for the model. The UI never sees this — it uses data.stdout.
|
||||
if (persistedOutputPath) {
|
||||
const preview = generatePreview(processedStdout, PREVIEW_SIZE_BYTES);
|
||||
// Prefer the bounded preview read from the full rolled-output source.
|
||||
// If that read failed, the in-memory value is only a captured head.
|
||||
const preview = persistedOutputPreview ?? generatePreview(
|
||||
processedStdout,
|
||||
PREVIEW_SIZE_BYTES,
|
||||
'head-only',
|
||||
).preview;
|
||||
const strategy = persistedOutputPreviewStrategy ?? 'head-only';
|
||||
processedStdout = buildLargeToolResultMessage({
|
||||
filepath: persistedOutputPath,
|
||||
originalSize: persistedOutputSize ?? 0,
|
||||
isJson: false,
|
||||
preview: preview.preview,
|
||||
hasMore: preview.hasMore,
|
||||
preview,
|
||||
hasMore: strategy !== 'complete',
|
||||
strategy,
|
||||
truncated: persistedOutputTruncated
|
||||
});
|
||||
}
|
||||
@@ -847,6 +903,9 @@ export const BashTool = buildTool({
|
||||
lastProgressFullOutput,
|
||||
);
|
||||
const outputWithSbFailures = SandboxManager.annotateStderrWithSandboxFailures(input.command, failureOutput);
|
||||
const sandboxDiagnostics = outputWithSbFailures.startsWith(failureOutput)
|
||||
? outputWithSbFailures.slice(failureOutput.length).trim()
|
||||
: '';
|
||||
if (result.preSpawnError) {
|
||||
throw new Error(result.preSpawnError);
|
||||
}
|
||||
@@ -865,18 +924,43 @@ export const BashTool = buildTool({
|
||||
// the truncated chunk and has no signal that the rest exists. The
|
||||
// persist step is identical to the success-path block below; both
|
||||
// sites resolve the same `result.outputFilePath` / outputTaskId.
|
||||
let errorStdout = outputWithSbFailures
|
||||
const errorExtraction = extractClaudeCodeHints(
|
||||
outputWithSbFailures,
|
||||
input.command,
|
||||
)
|
||||
let errorStdout = errorExtraction.stripped
|
||||
if (isMainThread) {
|
||||
for (const hint of errorExtraction.hints) {
|
||||
maybeRecordPluginHint(hint)
|
||||
}
|
||||
}
|
||||
if (result.outputFilePath && result.outputTaskId) {
|
||||
const persistedForError = await persistShellOutputFile(
|
||||
result.outputFilePath,
|
||||
result.outputTaskId,
|
||||
MAX_PERSISTED_SHELL_OUTPUT_SIZE,
|
||||
input.command,
|
||||
)
|
||||
if (persistedForError) {
|
||||
if (isMainThread) {
|
||||
for (const hint of persistedForError.previewHints ?? []) {
|
||||
const alreadyCaptured = errorExtraction.hints.some(
|
||||
captured =>
|
||||
captured.v === hint.v &&
|
||||
captured.type === hint.type &&
|
||||
captured.value === hint.value,
|
||||
)
|
||||
if (!alreadyCaptured) maybeRecordPluginHint(hint)
|
||||
}
|
||||
}
|
||||
errorStdout = appendPersistedOutputHint(
|
||||
errorStdout,
|
||||
persistedForError.path,
|
||||
persistedForError.size,
|
||||
persistedForError.truncated,
|
||||
persistedForError.preview,
|
||||
persistedForError.previewStrategy,
|
||||
sandboxDiagnostics,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -900,15 +984,23 @@ export const BashTool = buildTool({
|
||||
// FileRead. If > 64 MB, truncate after copying.
|
||||
let persistedOutputPath: string | undefined;
|
||||
let persistedOutputSize: number | undefined;
|
||||
let persistedOutputPreview: string | undefined;
|
||||
let persistedOutputPreviewStrategy: PreviewStrategy | undefined;
|
||||
let persistedOutputPreviewHints: ClaudeCodeHint[] = [];
|
||||
let persistedOutputTruncated: boolean | undefined;
|
||||
if (result.outputFilePath && result.outputTaskId) {
|
||||
const persisted = await persistShellOutputFile(
|
||||
result.outputFilePath,
|
||||
result.outputTaskId,
|
||||
MAX_PERSISTED_SHELL_OUTPUT_SIZE,
|
||||
input.command,
|
||||
);
|
||||
if (persisted) {
|
||||
persistedOutputPath = persisted.path;
|
||||
persistedOutputSize = persisted.size;
|
||||
persistedOutputPreview = persisted.preview;
|
||||
persistedOutputPreviewStrategy = persisted.previewStrategy;
|
||||
persistedOutputPreviewHints = persisted.previewHints ?? [];
|
||||
persistedOutputTruncated = persisted.truncated;
|
||||
}
|
||||
}
|
||||
@@ -950,6 +1042,17 @@ export const BashTool = buildTool({
|
||||
if (isMainThread && extracted.hints.length > 0) {
|
||||
for (const hint of extracted.hints) maybeRecordPluginHint(hint);
|
||||
}
|
||||
if (isMainThread && persistedOutputPreviewHints.length > 0) {
|
||||
for (const hint of persistedOutputPreviewHints) {
|
||||
const alreadyCaptured = extracted.hints.some(
|
||||
captured =>
|
||||
captured.v === hint.v &&
|
||||
captured.type === hint.type &&
|
||||
captured.value === hint.value,
|
||||
);
|
||||
if (!alreadyCaptured) maybeRecordPluginHint(hint);
|
||||
}
|
||||
}
|
||||
let isImage = isImageOutput(strippedStdout);
|
||||
|
||||
// Cap image dimensions + size if present (CC-304 — see
|
||||
@@ -984,6 +1087,8 @@ export const BashTool = buildTool({
|
||||
dangerouslyDisableSandbox: 'dangerouslyDisableSandbox' in input ? input.dangerouslyDisableSandbox as boolean | undefined : undefined,
|
||||
persistedOutputPath,
|
||||
persistedOutputSize,
|
||||
persistedOutputPreview,
|
||||
persistedOutputPreviewStrategy,
|
||||
persistedOutputTruncated
|
||||
};
|
||||
return {
|
||||
|
||||
@@ -10,12 +10,74 @@ import {
|
||||
import { tmpdir } from 'os'
|
||||
import { join } from 'path'
|
||||
import {
|
||||
PowerShellTool,
|
||||
appendPersistedPowerShellOutputHint,
|
||||
MAX_PERSISTED_POWERSHELL_OUTPUT_SIZE,
|
||||
persistPowerShellOutputFile,
|
||||
} from './PowerShellTool.js'
|
||||
import {
|
||||
generatePreview,
|
||||
PREVIEW_SIZE_BYTES,
|
||||
} from '../../utils/toolResultStorage.js'
|
||||
|
||||
describe('PowerShellTool persisted error output', () => {
|
||||
test('uses the persisted file preview for the model-facing success result', () => {
|
||||
const fullOutput = `COMMAND CONTEXT\n${'routine output\n'.repeat(300)}FAILURE ROOT\n`
|
||||
const preview = generatePreview(fullOutput, PREVIEW_SIZE_BYTES).preview
|
||||
const mapped = PowerShellTool.mapToolResultToToolResultBlockParam(
|
||||
{
|
||||
stdout: 'captured head only',
|
||||
stderr: '',
|
||||
interrupted: false,
|
||||
persistedOutputPath: '/tmp/full-output.txt',
|
||||
persistedOutputSize: 42_100,
|
||||
persistedOutputPreview: preview,
|
||||
persistedOutputPreviewStrategy: 'head-tail',
|
||||
} as never,
|
||||
'toolu_persisted_preview',
|
||||
)
|
||||
|
||||
expect(String(mapped.content)).toContain(preview)
|
||||
expect(String(mapped.content)).toContain('UTF-8-safe head and tail')
|
||||
expect(String(mapped.content)).not.toContain('complete available inline output')
|
||||
expect(Buffer.byteLength(preview, 'utf8')).toBeLessThanOrEqual(
|
||||
PREVIEW_SIZE_BYTES,
|
||||
)
|
||||
})
|
||||
|
||||
test('labels a small captured-only fallback as partial, not complete', () => {
|
||||
const mapped = PowerShellTool.mapToolResultToToolResultBlockParam(
|
||||
{
|
||||
stdout: 'small captured head',
|
||||
stderr: '',
|
||||
interrupted: false,
|
||||
persistedOutputPath: '/tmp/full-output.txt',
|
||||
persistedOutputSize: 42_100,
|
||||
} as never,
|
||||
'toolu_captured_fallback',
|
||||
)
|
||||
|
||||
expect(String(mapped.content)).toContain('UTF-8-safe head-only partial output')
|
||||
expect(String(mapped.content)).not.toContain('complete available inline output')
|
||||
})
|
||||
|
||||
test('under-claims a supplied preview when its strategy is missing', () => {
|
||||
const mapped = PowerShellTool.mapToolResultToToolResultBlockParam(
|
||||
{
|
||||
stdout: 'captured head only',
|
||||
stderr: '',
|
||||
interrupted: false,
|
||||
persistedOutputPath: '/tmp/full-output.txt',
|
||||
persistedOutputSize: 42_100,
|
||||
persistedOutputPreview: 'preview with unknown provenance',
|
||||
} as never,
|
||||
'toolu_preview_without_strategy',
|
||||
)
|
||||
|
||||
expect(String(mapped.content)).toContain('UTF-8-safe head-only partial output')
|
||||
expect(String(mapped.content)).not.toContain('UTF-8-safe head and tail')
|
||||
})
|
||||
|
||||
test('hint reports a cap instead of "full output" when the roll file was truncated', () => {
|
||||
const original = MAX_PERSISTED_POWERSHELL_OUTPUT_SIZE + 4096
|
||||
const hint = appendPersistedPowerShellOutputHint('preview', '/tmp/out', original, true)
|
||||
@@ -27,11 +89,55 @@ describe('PowerShellTool persisted error output', () => {
|
||||
expect(hint).toContain('/tmp/out')
|
||||
})
|
||||
|
||||
test('capped hint distinguishes preview tail bytes from saved bytes', () => {
|
||||
const hint = appendPersistedPowerShellOutputHint(
|
||||
'captured output',
|
||||
'/tmp/out',
|
||||
MAX_PERSISTED_POWERSHELL_OUTPUT_SIZE + 4096,
|
||||
true,
|
||||
'COMMAND CONTEXT\n… 4096 bytes omitted …\nFAILURE ROOT',
|
||||
'head-tail',
|
||||
)
|
||||
|
||||
expect(hint).toContain('preview may include tail bytes not saved at that path')
|
||||
})
|
||||
|
||||
test('hint keeps "full output" wording when the roll file fit under the cap', () => {
|
||||
const hint = appendPersistedPowerShellOutputHint('preview', '/tmp/out', 1234, false)
|
||||
expect(hint).toMatch(/full output \(1234 bytes\) saved to \/tmp\/out; read with the Read tool/)
|
||||
})
|
||||
|
||||
test('bounded error preview replaces the captured head', () => {
|
||||
const captured = 'captured duplicate\n'.repeat(2_000)
|
||||
const preview = 'COMMAND CONTEXT\n… 42,000 bytes omitted …\nFAILURE ROOT'
|
||||
const hint = appendPersistedPowerShellOutputHint(
|
||||
captured,
|
||||
'/tmp/out',
|
||||
42_100,
|
||||
false,
|
||||
preview,
|
||||
'head-tail',
|
||||
)
|
||||
|
||||
expect(hint).toContain(preview)
|
||||
expect(hint).not.toContain('captured duplicate')
|
||||
expect(Buffer.byteLength(hint, 'utf8')).toBeLessThan(3_000)
|
||||
})
|
||||
|
||||
test('labels a head-only persisted preview honestly', () => {
|
||||
const hint = appendPersistedPowerShellOutputHint(
|
||||
'captured output',
|
||||
'/tmp/out',
|
||||
42_100,
|
||||
false,
|
||||
'COMMAND CONTEXT',
|
||||
'head-only',
|
||||
)
|
||||
|
||||
expect(hint).toContain('UTF-8-safe head-only partial')
|
||||
expect(hint).not.toContain('UTF-8-safe head and tail')
|
||||
})
|
||||
|
||||
test('caps the destination copy and leaves the rolled-output source intact', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'powershell-persist-source-'))
|
||||
const source = join(dir, 'roll.txt')
|
||||
@@ -69,6 +175,12 @@ describe('PowerShellTool persisted error output', () => {
|
||||
expect(persisted).not.toBeNull()
|
||||
dest = persisted!.path
|
||||
expect(persisted!.truncated).toBe(true)
|
||||
expect(persisted!.preview).toStartWith('A')
|
||||
expect(persisted!.preview).toEndWith('B'.repeat(790))
|
||||
expect(persisted!.preview).toMatch(/… \d+ bytes omitted …/)
|
||||
expect(Buffer.byteLength(persisted!.preview!, 'utf8')).toBeLessThanOrEqual(
|
||||
PREVIEW_SIZE_BYTES,
|
||||
)
|
||||
const saved = readFileSync(dest, 'utf8')
|
||||
expect(saved.length).toBe(cap)
|
||||
expect(saved).toBe(head)
|
||||
@@ -78,4 +190,92 @@ describe('PowerShellTool persisted error output', () => {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('strips and reports a retained-tail Claude Code hint without changing the saved file', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'powershell-persist-hint-'))
|
||||
const source = join(dir, 'roll.txt')
|
||||
const hint =
|
||||
'<claude-code-hint v="1" type="plugin" value="example@claude-plugins-official" />'
|
||||
const body = `COMMAND CONTEXT\n${'routine output\n'.repeat(300)}${hint}\nFAILURE ROOT\n`
|
||||
writeFileSync(source, body)
|
||||
let dest: string | undefined
|
||||
|
||||
try {
|
||||
const persisted = await persistPowerShellOutputFile(
|
||||
source,
|
||||
'powershell-persist-hint-test',
|
||||
MAX_PERSISTED_POWERSHELL_OUTPUT_SIZE,
|
||||
'example-cli run',
|
||||
)
|
||||
expect(persisted).not.toBeNull()
|
||||
dest = persisted!.path
|
||||
expect(persisted!.preview).toContain('FAILURE ROOT')
|
||||
expect(persisted!.preview).not.toContain('<claude-code-hint')
|
||||
const marker = persisted!.preview!.match(/… (\d+) bytes omitted …/)
|
||||
expect(marker).toBeDefined()
|
||||
const displayedOutput = persisted!.preview!.replace(marker![0], '')
|
||||
expect(Number(marker![1])).toBe(
|
||||
Buffer.byteLength(body, 'utf8') -
|
||||
Buffer.byteLength(displayedOutput, 'utf8'),
|
||||
)
|
||||
expect(persisted!.previewHints).toEqual([
|
||||
{
|
||||
v: 1,
|
||||
type: 'plugin',
|
||||
value: 'example@claude-plugins-official',
|
||||
sourceCommand: 'example-cli',
|
||||
},
|
||||
])
|
||||
expect(readFileSync(dest, 'utf8')).toBe(body)
|
||||
} finally {
|
||||
if (dest && existsSync(dest)) rmSync(dest, { force: true })
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('downgrades a complete preview when a hint line is removed', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'powershell-persist-complete-hint-'))
|
||||
const source = join(dir, 'roll.txt')
|
||||
const hint =
|
||||
'<claude-code-hint v="1" type="plugin" value="example@claude-plugins-official" />'
|
||||
const body = `COMMAND CONTEXT\n${hint}\nFAILURE ROOT\n`
|
||||
writeFileSync(source, body)
|
||||
let dest: string | undefined
|
||||
|
||||
try {
|
||||
const persisted = await persistPowerShellOutputFile(
|
||||
source,
|
||||
'powershell-persist-complete-hint-test',
|
||||
MAX_PERSISTED_POWERSHELL_OUTPUT_SIZE,
|
||||
'example-cli run',
|
||||
)
|
||||
expect(persisted).not.toBeNull()
|
||||
dest = persisted!.path
|
||||
expect(persisted!.previewStrategy).toBe('head-only')
|
||||
expect(persisted!.preview).not.toContain('<claude-code-hint')
|
||||
|
||||
const mapped = PowerShellTool.mapToolResultToToolResultBlockParam(
|
||||
{
|
||||
stdout: 'captured head only',
|
||||
stderr: '',
|
||||
interrupted: false,
|
||||
persistedOutputPath: persisted!.path,
|
||||
persistedOutputSize: persisted!.size,
|
||||
persistedOutputPreview: persisted!.preview,
|
||||
persistedOutputPreviewStrategy: persisted!.previewStrategy,
|
||||
} as never,
|
||||
'toolu_sanitized_complete_preview',
|
||||
)
|
||||
expect(String(mapped.content)).toContain(
|
||||
'UTF-8-safe head-only partial output',
|
||||
)
|
||||
expect(String(mapped.content)).not.toContain(
|
||||
'complete available inline output',
|
||||
)
|
||||
expect(readFileSync(dest, 'utf8')).toBe(body)
|
||||
} finally {
|
||||
if (dest && existsSync(dest)) rmSync(dest, { force: true })
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -15,7 +15,7 @@ import { buildTool, type ToolDef } from '../../Tool.js';
|
||||
import { backgroundExistingForegroundTask, markTaskNotified, registerForeground, spawnShellTask, unregisterForeground } from '../../tasks/LocalShellTask/LocalShellTask.js';
|
||||
import type { AgentId } from '../../types/ids.js';
|
||||
import type { AssistantMessage } from '../../types/message.js';
|
||||
import { extractClaudeCodeHints } from '../../utils/claudeCodeHints.js';
|
||||
import { extractClaudeCodeHints, extractClaudeCodeHintsFromPreview, type ClaudeCodeHint } from '../../utils/claudeCodeHints.js';
|
||||
import { isEnvTruthy } from '../../utils/envUtils.js';
|
||||
import { errorMessage as getErrorMessage, ShellError } from '../../utils/errors.js';
|
||||
import { truncate } from '../../utils/format.js';
|
||||
@@ -35,7 +35,7 @@ import { EndTruncatingAccumulator } from '../../utils/stringUtils.js';
|
||||
import { getTaskOutputPath } from '../../utils/task/diskOutput.js';
|
||||
import { TaskOutput } from '../../utils/task/TaskOutput.js';
|
||||
import { isOutputLineTruncated } from '../../utils/terminal.js';
|
||||
import { buildLargeToolResultMessage, ensureToolResultsDir, generatePreview, getToolResultPath, PREVIEW_SIZE_BYTES } from '../../utils/toolResultStorage.js';
|
||||
import { buildLargeToolResultMessage, ensureToolResultsDir, generateFilePreview, generatePreview, getToolResultPath, PREVIEW_SIZE_BYTES, type PreviewStrategy } from '../../utils/toolResultStorage.js';
|
||||
import { shouldUseSandbox } from '../BashTool/shouldUseSandbox.js';
|
||||
import { BackgroundHint } from '../BashTool/UI.js';
|
||||
import { buildImageToolResult, isImageOutput, resetCwdIfOutsideProject, resizeShellImageOutput, stdErrAppendShellResetMessage, stripEmptyLines } from '../BashTool/utils.js';
|
||||
@@ -55,7 +55,8 @@ export async function persistPowerShellOutputFile(
|
||||
sourcePath: string,
|
||||
taskId: string,
|
||||
maxSize: number = MAX_PERSISTED_POWERSHELL_OUTPUT_SIZE,
|
||||
): Promise<{ path: string; size: number; truncated: boolean } | null> {
|
||||
command: string = '',
|
||||
): Promise<{ path: string; size: number; truncated: boolean; preview?: string; previewStrategy?: PreviewStrategy; previewHints?: ClaudeCodeHint[] } | null> {
|
||||
try {
|
||||
const fileStat = await fsStat(sourcePath);
|
||||
const size = fileStat.size;
|
||||
@@ -79,7 +80,25 @@ export async function persistPowerShellOutputFile(
|
||||
await copyFile(sourcePath, dest);
|
||||
}
|
||||
}
|
||||
return { path: dest, size, truncated };
|
||||
const previewSourcePath = truncated ? sourcePath : dest;
|
||||
const previewResult = await generateFilePreview(
|
||||
previewSourcePath,
|
||||
PREVIEW_SIZE_BYTES,
|
||||
).catch(error => {
|
||||
logError(error instanceof Error ? error : new Error(getErrorMessage(error)));
|
||||
return undefined;
|
||||
});
|
||||
const previewExtraction = previewResult
|
||||
? extractClaudeCodeHintsFromPreview(previewResult, command)
|
||||
: undefined;
|
||||
return {
|
||||
path: dest,
|
||||
size,
|
||||
truncated,
|
||||
preview: previewExtraction?.previewResult.preview,
|
||||
previewStrategy: previewExtraction?.previewResult.strategy,
|
||||
previewHints: previewExtraction?.hints,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
@@ -90,13 +109,28 @@ export function appendPersistedPowerShellOutputHint(
|
||||
persistedPath: string,
|
||||
persistedSize: number,
|
||||
truncated: boolean,
|
||||
preview?: string,
|
||||
previewStrategy?: PreviewStrategy,
|
||||
): string {
|
||||
const capDetail = previewStrategy === 'head-tail'
|
||||
? 'capped; preview may include tail bytes not saved at that path'
|
||||
: 'capped, tail not saved';
|
||||
const hint = truncated
|
||||
? `[output truncated above — first ${MAX_PERSISTED_POWERSHELL_OUTPUT_SIZE} bytes of the ${persistedSize}-byte output saved to ${persistedPath} (capped, tail not saved); read with the Read tool]`
|
||||
? `[output truncated above — first ${MAX_PERSISTED_POWERSHELL_OUTPUT_SIZE} bytes of the ${persistedSize}-byte output saved to ${persistedPath} (${capDetail}); read with the Read tool]`
|
||||
: `[output truncated above — full output (${persistedSize} bytes) saved to ${persistedPath}; read with the Read tool]`;
|
||||
if (!stdout) return hint;
|
||||
const trimmed = stdout.endsWith('\n') ? stdout.slice(0, -1) : stdout;
|
||||
return `${trimmed}\n\n${hint}`;
|
||||
const previewStrategyLabel = previewStrategy === 'head-tail'
|
||||
? 'head and tail'
|
||||
: previewStrategy === 'complete'
|
||||
? 'complete'
|
||||
: 'head-only partial';
|
||||
const previewBlock = preview
|
||||
? `Persisted output preview (UTF-8-safe ${previewStrategyLabel}, ${PREVIEW_SIZE_BYTES}-byte budget):\n${preview}`
|
||||
: '';
|
||||
const capturedFallback = preview
|
||||
? ''
|
||||
: stdout.endsWith('\n') ? stdout.slice(0, -1) : stdout;
|
||||
const parts = [capturedFallback, previewBlock, hint].filter(Boolean);
|
||||
return parts.join('\n\n');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -512,6 +546,8 @@ const outputSchema = lazySchema(() => z.object({
|
||||
isImage: z.boolean().optional().describe('Flag to indicate if stdout contains image data'),
|
||||
persistedOutputPath: z.string().optional().describe('Path to persisted full output when too large for inline'),
|
||||
persistedOutputSize: z.number().optional().describe('Total output size in bytes when persisted'),
|
||||
persistedOutputPreview: z.string().optional().describe('UTF-8-safe head/tail preview read from the complete rolled-output source'),
|
||||
persistedOutputPreviewStrategy: z.enum(['complete', 'head-tail', 'head-only']).optional().describe('How the persisted output preview was selected'),
|
||||
persistedOutputTruncated: z.boolean().optional().describe('Whether the persisted file is capped (only the first portion of the output was saved)'),
|
||||
backgroundTaskId: z.string().optional().describe('ID of the background task if command is running in background'),
|
||||
backgroundedByUser: z.boolean().optional().describe('True if the user manually backgrounded the command with Ctrl+B'),
|
||||
@@ -652,6 +688,8 @@ export const PowerShellTool = buildTool({
|
||||
isImage,
|
||||
persistedOutputPath,
|
||||
persistedOutputSize,
|
||||
persistedOutputPreview,
|
||||
persistedOutputPreviewStrategy,
|
||||
persistedOutputTruncated,
|
||||
backgroundTaskId,
|
||||
backgroundedByUser,
|
||||
@@ -670,13 +708,21 @@ export const PowerShellTool = buildTool({
|
||||
const trimmed = normalizedStdout
|
||||
? normalizedStdout.replace(/^(\s*\n)+/, '').trimEnd()
|
||||
: '';
|
||||
const preview = generatePreview(trimmed, PREVIEW_SIZE_BYTES);
|
||||
// Prefer the bounded preview read from the full rolled-output source.
|
||||
// If that read failed, the in-memory value is only a captured head.
|
||||
const preview = persistedOutputPreview ?? generatePreview(
|
||||
trimmed,
|
||||
PREVIEW_SIZE_BYTES,
|
||||
'head-only',
|
||||
).preview;
|
||||
const strategy = persistedOutputPreviewStrategy ?? 'head-only';
|
||||
processedStdout = buildLargeToolResultMessage({
|
||||
filepath: persistedOutputPath,
|
||||
originalSize: persistedOutputSize ?? 0,
|
||||
isJson: false,
|
||||
preview: preview.preview,
|
||||
hasMore: preview.hasMore,
|
||||
preview,
|
||||
hasMore: strategy !== 'complete',
|
||||
strategy,
|
||||
truncated: persistedOutputTruncated
|
||||
});
|
||||
} else if (normalizedStdout) {
|
||||
@@ -858,13 +904,28 @@ export const PowerShellTool = buildTool({
|
||||
const persistedForError = await persistPowerShellOutputFile(
|
||||
result.outputFilePath,
|
||||
result.outputTaskId,
|
||||
MAX_PERSISTED_POWERSHELL_OUTPUT_SIZE,
|
||||
input.command,
|
||||
);
|
||||
if (persistedForError) {
|
||||
if (isMainThread) {
|
||||
for (const hint of persistedForError.previewHints ?? []) {
|
||||
const alreadyCaptured = extracted.hints.some(
|
||||
captured =>
|
||||
captured.v === hint.v &&
|
||||
captured.type === hint.type &&
|
||||
captured.value === hint.value,
|
||||
);
|
||||
if (!alreadyCaptured) maybeRecordPluginHint(hint);
|
||||
}
|
||||
}
|
||||
errorStdout = appendPersistedPowerShellOutputHint(
|
||||
errorStdout,
|
||||
persistedForError.path,
|
||||
persistedForError.size,
|
||||
persistedForError.truncated,
|
||||
persistedForError.preview,
|
||||
persistedForError.previewStrategy,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -880,16 +941,33 @@ export const PowerShellTool = buildTool({
|
||||
// tool-results dir so the model can read it via FileRead.
|
||||
let persistedOutputPath: string | undefined;
|
||||
let persistedOutputSize: number | undefined;
|
||||
let persistedOutputPreview: string | undefined;
|
||||
let persistedOutputPreviewStrategy: PreviewStrategy | undefined;
|
||||
let persistedOutputTruncated: boolean | undefined;
|
||||
if (result.outputFilePath && result.outputTaskId) {
|
||||
const persisted = await persistPowerShellOutputFile(
|
||||
result.outputFilePath,
|
||||
result.outputTaskId,
|
||||
MAX_PERSISTED_POWERSHELL_OUTPUT_SIZE,
|
||||
input.command,
|
||||
);
|
||||
if (persisted) {
|
||||
persistedOutputPath = persisted.path;
|
||||
persistedOutputSize = persisted.size;
|
||||
persistedOutputPreview = persisted.preview;
|
||||
persistedOutputPreviewStrategy = persisted.previewStrategy;
|
||||
persistedOutputTruncated = persisted.truncated;
|
||||
if (isMainThread) {
|
||||
for (const hint of persisted.previewHints ?? []) {
|
||||
const alreadyCaptured = extracted.hints.some(
|
||||
captured =>
|
||||
captured.v === hint.v &&
|
||||
captured.type === hint.type &&
|
||||
captured.value === hint.value,
|
||||
);
|
||||
if (!alreadyCaptured) maybeRecordPluginHint(hint);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -937,6 +1015,8 @@ export const PowerShellTool = buildTool({
|
||||
isImage,
|
||||
persistedOutputPath,
|
||||
persistedOutputSize,
|
||||
persistedOutputPreview,
|
||||
persistedOutputPreviewStrategy,
|
||||
persistedOutputTruncated
|
||||
}
|
||||
};
|
||||
|
||||
@@ -17,6 +17,10 @@
|
||||
|
||||
import { logForDebugging } from './debug.js'
|
||||
import { createSignal } from './signal.js'
|
||||
import {
|
||||
formatOmissionMarker,
|
||||
type PreviewResult,
|
||||
} from './toolResultStorage.js'
|
||||
|
||||
export type ClaudeCodeHintType = 'plugin'
|
||||
|
||||
@@ -119,6 +123,75 @@ export function extractClaudeCodeHints(
|
||||
return { hints, stripped: collapsed }
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip harness-only hint lines while keeping preview completeness and exact
|
||||
* omitted-byte metadata aligned with the text that remains model-visible.
|
||||
*/
|
||||
export function extractClaudeCodeHintsFromPreview(
|
||||
result: PreviewResult,
|
||||
command: string,
|
||||
): { hints: ClaudeCodeHint[]; previewResult: PreviewResult } {
|
||||
if (
|
||||
result.strategy === 'head-tail' &&
|
||||
result.omittedBytes !== undefined &&
|
||||
result.markerStart !== undefined
|
||||
) {
|
||||
const marker = formatOmissionMarker(result.omittedBytes)
|
||||
const markerEnd = result.markerStart + marker.length
|
||||
if (result.preview.slice(result.markerStart, markerEnd) === marker) {
|
||||
const head = result.preview.slice(0, result.markerStart)
|
||||
const tail = result.preview.slice(markerEnd)
|
||||
const headExtraction = extractClaudeCodeHints(head, command)
|
||||
const tailExtraction = extractClaudeCodeHints(tail, command)
|
||||
const removedBytes =
|
||||
Buffer.byteLength(head, 'utf8') +
|
||||
Buffer.byteLength(tail, 'utf8') -
|
||||
Buffer.byteLength(headExtraction.stripped, 'utf8') -
|
||||
Buffer.byteLength(tailExtraction.stripped, 'utf8')
|
||||
if (removedBytes > 0 && result.retainedBytesValidUtf8 === false) {
|
||||
return {
|
||||
hints: [...headExtraction.hints, ...tailExtraction.hints],
|
||||
previewResult: {
|
||||
preview: headExtraction.stripped,
|
||||
hasMore: true,
|
||||
strategy: 'head-only',
|
||||
retainedBytesValidUtf8: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
const omittedBytes = result.omittedBytes + removedBytes
|
||||
const nextMarker = formatOmissionMarker(omittedBytes)
|
||||
return {
|
||||
hints: [...headExtraction.hints, ...tailExtraction.hints],
|
||||
previewResult: {
|
||||
...result,
|
||||
preview:
|
||||
headExtraction.stripped +
|
||||
nextMarker +
|
||||
tailExtraction.stripped,
|
||||
omittedBytes,
|
||||
markerStart: headExtraction.stripped.length,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const extraction = extractClaudeCodeHints(result.preview, command)
|
||||
const removedContent = extraction.stripped !== result.preview
|
||||
return {
|
||||
hints: extraction.hints,
|
||||
previewResult: {
|
||||
...result,
|
||||
preview: extraction.stripped,
|
||||
hasMore: result.hasMore || removedContent,
|
||||
strategy:
|
||||
removedContent && result.strategy === 'complete'
|
||||
? 'head-only'
|
||||
: result.strategy,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function parseAttrs(tagBody: string): Record<string, string> {
|
||||
const attrs: Record<string, string> = {}
|
||||
for (const m of tagBody.matchAll(ATTR_RE)) {
|
||||
|
||||
@@ -31,19 +31,19 @@ export function getFormatDescription(
|
||||
* Generates instruction text for Claude to read from a saved output file.
|
||||
*
|
||||
* @param rawOutputPath - Path to the saved output file
|
||||
* @param contentLength - Length of the content in characters
|
||||
* @param contentSizeBytes - UTF-8 byte size of the content
|
||||
* @param formatDescription - Description of the content format
|
||||
* @param maxReadLength - Optional max chars for Read tool (for Bash output context)
|
||||
* @returns Instruction text to include in the tool result
|
||||
*/
|
||||
export function getLargeOutputInstructions(
|
||||
rawOutputPath: string,
|
||||
contentLength: number,
|
||||
contentSizeBytes: number,
|
||||
formatDescription: string,
|
||||
maxReadLength?: number,
|
||||
): string {
|
||||
const baseInstructions =
|
||||
`Error: result (${contentLength.toLocaleString()} characters) exceeds maximum allowed tokens. Output has been saved to ${rawOutputPath}.\n` +
|
||||
`Error: result (${contentSizeBytes.toLocaleString()} bytes) exceeds maximum allowed tokens. Output has been saved to ${rawOutputPath}.\n` +
|
||||
`Format: ${formatDescription}\n` +
|
||||
`Use offset and limit parameters to read specific portions of the file, search within it for specific content, and jq to make structured queries.\n` +
|
||||
`REQUIREMENTS FOR SUMMARIZATION/ANALYSIS/REVIEW:\n` +
|
||||
@@ -58,6 +58,14 @@ export function getLargeOutputInstructions(
|
||||
return baseInstructions + truncationWarning + completionRequirement
|
||||
}
|
||||
|
||||
export function getLargeOutputPersistenceFailureInstructions(
|
||||
content: string,
|
||||
error: string,
|
||||
): string {
|
||||
const contentSizeBytes = Buffer.byteLength(content, 'utf8')
|
||||
return `Error: result (${contentSizeBytes.toLocaleString()} bytes) exceeds maximum allowed tokens. Failed to save output to file: ${error}. If this MCP server provides pagination or filtering tools, use them to retrieve specific portions of the data.`
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a mime type to a file extension. Conservative: known types get their
|
||||
* proper extension; unknown types get 'bin'. The extension matters because
|
||||
|
||||
@@ -0,0 +1,491 @@
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'fs/promises'
|
||||
import { tmpdir } from 'os'
|
||||
import { join } from 'path'
|
||||
|
||||
import {
|
||||
getOriginalCwd,
|
||||
getSessionId,
|
||||
setOriginalCwd,
|
||||
switchSession,
|
||||
} from '../bootstrap/state.ts'
|
||||
import type { SessionId } from '../types/ids.ts'
|
||||
import {
|
||||
getClaudeConfigHomeDirOverrideForTesting,
|
||||
setClaudeConfigHomeDirForTesting,
|
||||
} from './envUtils.ts'
|
||||
import { formatFileSize } from './format.ts'
|
||||
import {
|
||||
getLargeOutputInstructions,
|
||||
getLargeOutputPersistenceFailureInstructions,
|
||||
} from './mcpOutputStorage.ts'
|
||||
import { createUserMessage } from './messages.ts'
|
||||
import { jsonStringify } from './slowOperations.ts'
|
||||
import {
|
||||
applyToolResultReplacementsToMessages,
|
||||
buildLargeToolResultMessage,
|
||||
formatOmissionMarker,
|
||||
generateFilePreview,
|
||||
generatePreview,
|
||||
isPersistError,
|
||||
persistToolResult,
|
||||
PREVIEW_SIZE_BYTES,
|
||||
reconstructContentReplacementState,
|
||||
} from './toolResultStorage.ts'
|
||||
|
||||
const byteLength = (value: string): number => Buffer.byteLength(value, 'utf8')
|
||||
|
||||
function expectWithinBudget(value: string, maxBytes: number): void {
|
||||
expect(byteLength(value)).toBeLessThanOrEqual(maxBytes)
|
||||
expect(value).not.toContain('\uFFFD')
|
||||
}
|
||||
|
||||
function expectExactOmittedByteCount(
|
||||
content: string,
|
||||
preview: string,
|
||||
): void {
|
||||
const matches = [...preview.matchAll(/… (\d+) bytes omitted …/g)]
|
||||
const match = matches.find(candidate => {
|
||||
const retainedBytes = byteLength(preview) - byteLength(candidate[0])
|
||||
return Number(candidate[1]) === byteLength(content) - retainedBytes
|
||||
})
|
||||
expect(match).toBeDefined()
|
||||
const marker = match![0]
|
||||
const retainedBytes = byteLength(preview) - byteLength(marker)
|
||||
expect(Number(match![1])).toBe(byteLength(content) - retainedBytes)
|
||||
}
|
||||
|
||||
describe('generatePreview UTF-8 byte accounting', () => {
|
||||
test.each([
|
||||
['just below', 'a'.repeat(63)],
|
||||
['exactly at', 'a'.repeat(64)],
|
||||
])('returns ASCII %s the limit unchanged', (_label, content) => {
|
||||
expect(generatePreview(content, 64)).toEqual({
|
||||
preview: content,
|
||||
hasMore: false,
|
||||
strategy: 'complete',
|
||||
})
|
||||
})
|
||||
|
||||
test('truncates ASCII just above the limit within the byte budget', () => {
|
||||
const content = 'a'.repeat(65)
|
||||
const result = generatePreview(content, 64)
|
||||
|
||||
expect(result.hasMore).toBe(true)
|
||||
expect(result.strategy).toBe('head-tail')
|
||||
expectWithinBudget(result.preview, 64)
|
||||
expectExactOmittedByteCount(content, result.preview)
|
||||
})
|
||||
|
||||
test('uses UTF-8 bytes when CJK is under the UTF-16 count but over the limit', () => {
|
||||
const content = '界'.repeat(30)
|
||||
expect(content.length).toBeLessThan(80)
|
||||
expect(byteLength(content)).toBeGreaterThan(80)
|
||||
|
||||
const result = generatePreview(content, 80)
|
||||
|
||||
expect(result.hasMore).toBe(true)
|
||||
expectWithinBudget(result.preview, 80)
|
||||
expectExactOmittedByteCount(content, result.preview)
|
||||
})
|
||||
|
||||
test('does not corrupt emoji or combining sequences at byte boundaries', () => {
|
||||
const content = '🙂e\u0301'.repeat(80)
|
||||
const result = generatePreview(content, 97)
|
||||
|
||||
expect(result.hasMore).toBe(true)
|
||||
expectWithinBudget(result.preview, 97)
|
||||
expectExactOmittedByteCount(content, result.preview)
|
||||
})
|
||||
|
||||
test('returns empty content unchanged', () => {
|
||||
expect(generatePreview('', 32)).toEqual({
|
||||
preview: '',
|
||||
hasMore: false,
|
||||
strategy: 'complete',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('generatePreview head and tail selection', () => {
|
||||
test('keeps command context and the only failure root while omitting the middle', () => {
|
||||
const content = [
|
||||
'$ bun run build',
|
||||
'Compiling packages...',
|
||||
...Array.from({ length: 200 }, (_, index) => `routine output ${index}`),
|
||||
'Error: build failed',
|
||||
'at decisiveStackRoot (/workspace/src/index.ts:42:7)',
|
||||
].join('\n')
|
||||
|
||||
const result = generatePreview(content, 240)
|
||||
|
||||
expect(result.preview).toStartWith('$ bun run build\n')
|
||||
expect(result.preview).toContain('Error: build failed')
|
||||
expect(result.preview).toContain(
|
||||
'at decisiveStackRoot (/workspace/src/index.ts:42:7)',
|
||||
)
|
||||
expect(result.preview).not.toContain('routine output 100')
|
||||
expectWithinBudget(result.preview, 240)
|
||||
expectExactOmittedByteCount(content, result.preview)
|
||||
})
|
||||
|
||||
test('uses complete-line boundaries for CRLF input', () => {
|
||||
const content = [
|
||||
'COMMAND context',
|
||||
...Array.from({ length: 80 }, (_, index) => `middle-${index}`),
|
||||
'FAILURE summary',
|
||||
'',
|
||||
].join('\r\n')
|
||||
|
||||
const result = generatePreview(content, 120)
|
||||
const markerIndex = result.preview.indexOf('… ')
|
||||
const afterMarker = result.preview.indexOf(' …') + ' …'.length
|
||||
|
||||
expect(result.preview.slice(0, markerIndex)).toEndWith('\r\n')
|
||||
expect(result.preview.slice(afterMarker)).toStartWith('\r\n')
|
||||
expect(result.preview).toContain('FAILURE summary\r\n')
|
||||
expectWithinBudget(result.preview, 120)
|
||||
})
|
||||
|
||||
test('falls back to UTF-8-safe hard cuts for one giant line', () => {
|
||||
const content = `HEAD-${'界🙂'.repeat(100)}-TAIL`
|
||||
const result = generatePreview(content, 96)
|
||||
|
||||
expect(result.preview).toStartWith('HEAD-')
|
||||
expect(result.preview).toEndWith('-TAIL')
|
||||
expectWithinBudget(result.preview, 96)
|
||||
expectExactOmittedByteCount(content, result.preview)
|
||||
})
|
||||
|
||||
test('uses newlines that fall exactly near the allocation targets', () => {
|
||||
// With this content size and budget, 28 bytes go to the head and 19 to
|
||||
// the tail. Put line breaks exactly at those selection boundaries.
|
||||
const headLine = `${'H'.repeat(27)}\n`
|
||||
const content = `${headLine}${'m'.repeat(300)}\nTAIL-LINE`
|
||||
const result = generatePreview(content, 72)
|
||||
const marker = result.preview.match(/… \d+ bytes omitted …/)![0]
|
||||
const [head, tail] = result.preview.split(marker)
|
||||
|
||||
expect(head).toBe(headLine)
|
||||
expect(tail).toBe('\nTAIL-LINE')
|
||||
expectWithinBudget(result.preview, 72)
|
||||
})
|
||||
|
||||
test('can spend a tiny budget entirely on the omission marker', () => {
|
||||
const content = 'x'.repeat(100)
|
||||
const marker = '… 100 bytes omitted …'
|
||||
const result = generatePreview(content, byteLength(marker))
|
||||
|
||||
expect(result.preview).toBe(marker)
|
||||
expect(result.hasMore).toBe(true)
|
||||
expect(result.strategy).toBe('head-tail')
|
||||
})
|
||||
|
||||
test('falls back to a bounded head-only preview when the marker cannot fit', () => {
|
||||
const content = 'x'.repeat(100)
|
||||
const result = generatePreview(content, 1)
|
||||
|
||||
expect(result.preview).toBe('x')
|
||||
expect(result.hasMore).toBe(true)
|
||||
expect(result.strategy).toBe('head-only')
|
||||
expectWithinBudget(result.preview, 1)
|
||||
})
|
||||
|
||||
test('does not duplicate content when the potential head and tail are close', () => {
|
||||
const content = '0123456789'.repeat(5)
|
||||
const result = generatePreview(content, 49)
|
||||
const match = result.preview.match(/… (\d+) bytes omitted …/)
|
||||
|
||||
expect(match).not.toBeNull()
|
||||
expect(Number(match![1])).toBeGreaterThan(0)
|
||||
expectWithinBudget(result.preview, 49)
|
||||
expectExactOmittedByteCount(content, result.preview)
|
||||
})
|
||||
|
||||
test('preserves a trailing newline in the retained tail', () => {
|
||||
const content = `head\n${'middle\n'.repeat(80)}failure summary\n`
|
||||
const result = generatePreview(content, 96)
|
||||
|
||||
expect(result.preview).toEndWith('failure summary\n')
|
||||
expectWithinBudget(result.preview, 96)
|
||||
})
|
||||
|
||||
test('reads a UTF-8-safe head and tail preview from a persisted file', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'tool-file-preview-'))
|
||||
const filepath = join(dir, 'large-output.txt')
|
||||
const content = `COMMAND CONTEXT\n${'routine 界 output\n'.repeat(300)}FAILURE ROOT: src/index.ts:42\n`
|
||||
await writeFile(filepath, content, 'utf8')
|
||||
|
||||
try {
|
||||
const result = await generateFilePreview(
|
||||
filepath,
|
||||
200,
|
||||
)
|
||||
|
||||
expect(result.strategy).toBe('head-tail')
|
||||
expect(result.preview).toStartWith('COMMAND CONTEXT\n')
|
||||
expect(result.preview).toContain('FAILURE ROOT: src/index.ts:42')
|
||||
expectWithinBudget(result.preview, 200)
|
||||
expectExactOmittedByteCount(content, result.preview)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test.each([
|
||||
['raw bytes over the limit', 3_000],
|
||||
['raw bytes under the limit that expand when decoded', 1_000],
|
||||
])('bounds malformed UTF-8 after decoding: %s', async (_label, size) => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'tool-file-preview-invalid-utf8-'))
|
||||
const filepath = join(dir, 'invalid-output.bin')
|
||||
await writeFile(filepath, Buffer.alloc(size, 0xff))
|
||||
|
||||
try {
|
||||
const result = await generateFilePreview(filepath, 2_000)
|
||||
const marker = result.preview.match(/… (\d+) bytes omitted …/)
|
||||
const retainedSourceBytes = [...result.preview].filter(
|
||||
character => character === '\uFFFD',
|
||||
).length
|
||||
|
||||
expect(Buffer.byteLength(result.preview, 'utf8')).toBeLessThanOrEqual(
|
||||
2_000,
|
||||
)
|
||||
expect(result.hasMore).toBe(true)
|
||||
expect(result.strategy).toBe('head-tail')
|
||||
expect(result.retainedBytesValidUtf8).toBe(false)
|
||||
expect(marker).not.toBeNull()
|
||||
expect(Number(marker![1])).toBe(size - retainedSourceBytes)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('derives the preview size from the opened file', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'tool-file-preview-size-'))
|
||||
const filepath = join(dir, 'large-output.txt')
|
||||
const content = `COMMAND CONTEXT\n${'routine output\n'.repeat(80)}FAILURE ROOT\n`
|
||||
await writeFile(filepath, content, 'utf8')
|
||||
|
||||
try {
|
||||
const result = await generateFilePreview(filepath, 96)
|
||||
|
||||
expect(result.strategy).toBe('head-tail')
|
||||
expect(result.preview).toStartWith('COMMAND CONTEXT\n')
|
||||
expect(result.preview).toContain('FAILURE ROOT')
|
||||
expectWithinBudget(result.preview, 96)
|
||||
expectExactOmittedByteCount(content, result.preview)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('reports actual omitted bytes for a short file', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'tool-file-preview-stale-size-'))
|
||||
const filepath = join(dir, 'large-output.txt')
|
||||
const content = `HEAD\n${'middle\n'.repeat(30)}REAL TAIL\n`
|
||||
await writeFile(filepath, content, 'utf8')
|
||||
|
||||
try {
|
||||
const result = await generateFilePreview(filepath, 96)
|
||||
|
||||
expect(result.preview).toContain('REAL TAIL')
|
||||
expectWithinBudget(result.preview, 96)
|
||||
expectExactOmittedByteCount(content, result.preview)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('distinguishes a generated omission marker from marker-like output', () => {
|
||||
const content = [
|
||||
'COMMAND: printf "… 1 bytes omitted …"',
|
||||
...Array.from({ length: 100 }, (_, index) => `middle-${index}`),
|
||||
'REAL TAIL',
|
||||
].join('\n')
|
||||
const result = generatePreview(content, 120)
|
||||
|
||||
expect(result.preview).toContain('… 1 bytes omitted …')
|
||||
expect(result.preview).toContain('REAL TAIL')
|
||||
expectExactOmittedByteCount(content, result.preview)
|
||||
expect(result.omittedBytes).toBeDefined()
|
||||
expect(result.markerStart).toBeDefined()
|
||||
const generatedMarker = formatOmissionMarker(result.omittedBytes!)
|
||||
expect(
|
||||
result.preview.slice(
|
||||
result.markerStart!,
|
||||
result.markerStart! + generatedMarker.length,
|
||||
),
|
||||
).toBe(generatedMarker)
|
||||
})
|
||||
})
|
||||
|
||||
describe('generatePreview serialized JSON policy', () => {
|
||||
test('returns small JSON unchanged', () => {
|
||||
const content = '{"ok":"界"}'
|
||||
expect(generatePreview(content, 64, 'json')).toEqual({
|
||||
preview: content,
|
||||
hasMore: false,
|
||||
strategy: 'complete',
|
||||
})
|
||||
})
|
||||
|
||||
test('uses an explicitly head-only partial fragment for large JSON', () => {
|
||||
const content = jsonStringify(
|
||||
{
|
||||
beginning: 'JSON_BEGIN',
|
||||
middle: '界'.repeat(200),
|
||||
ending: 'JSON_TAIL_MUST_NOT_APPEAR',
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)
|
||||
|
||||
const result = generatePreview(content, 96, 'json')
|
||||
|
||||
expect(result.strategy).toBe('head-only')
|
||||
expect(result.hasMore).toBe(true)
|
||||
expect(result.preview).toContain('JSON_BEGIN')
|
||||
expect(result.preview).not.toContain('JSON_TAIL_MUST_NOT_APPEAR')
|
||||
expectWithinBudget(result.preview, 96)
|
||||
})
|
||||
})
|
||||
|
||||
test('supports an honest head-only strategy when only an initial output chunk is available', () => {
|
||||
const content = `CAPTURED_HEAD\n${'middle\n'.repeat(80)}CAPTURED_CHUNK_TAIL`
|
||||
const result = generatePreview(content, 96, 'head-only')
|
||||
|
||||
expect(result.strategy).toBe('head-only')
|
||||
expect(result.preview).toContain('CAPTURED_HEAD')
|
||||
expect(result.preview).not.toContain('CAPTURED_CHUNK_TAIL')
|
||||
expectWithinBudget(result.preview, 96)
|
||||
})
|
||||
|
||||
describe('persisted tool-result preview integration', () => {
|
||||
let tempConfigDir: string
|
||||
let previousConfigDir: string | undefined
|
||||
let previousCwd: string
|
||||
let previousSessionId: SessionId
|
||||
|
||||
beforeAll(async () => {
|
||||
tempConfigDir = await mkdtemp(join(tmpdir(), 'tool-preview-'))
|
||||
previousConfigDir = getClaudeConfigHomeDirOverrideForTesting()
|
||||
previousCwd = getOriginalCwd()
|
||||
previousSessionId = getSessionId()
|
||||
setClaudeConfigHomeDirForTesting(tempConfigDir)
|
||||
setOriginalCwd(join(tempConfigDir, 'workspace'))
|
||||
switchSession('tool-preview-session' as SessionId)
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
switchSession(previousSessionId)
|
||||
setOriginalCwd(previousCwd)
|
||||
setClaudeConfigHomeDirForTesting(previousConfigDir)
|
||||
await rm(tempConfigDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test('keeps the full plain-text spill, reports UTF-8 bytes, and replays EEXIST deterministically', async () => {
|
||||
const content = `COMMAND: bun test\n${'routine 界 output\n'.repeat(300)}FAILURE ROOT: src/index.ts:42\n`
|
||||
const first = await persistToolResult(content, 'plain-text-preview')
|
||||
expect(isPersistError(first)).toBe(false)
|
||||
if (isPersistError(first)) throw new Error(first.error)
|
||||
|
||||
expect(await readFile(first.filepath, 'utf8')).toBe(content)
|
||||
expect(first.originalSize).toBe(byteLength(content))
|
||||
expect(first.hasMore).toBe(true)
|
||||
expect(first.strategy).toBe('head-tail')
|
||||
expect(first.preview).toContain('COMMAND: bun test')
|
||||
expect(first.preview).toContain('FAILURE ROOT: src/index.ts:42')
|
||||
expectWithinBudget(first.preview, PREVIEW_SIZE_BYTES)
|
||||
|
||||
const replay = await persistToolResult(content, 'plain-text-preview')
|
||||
expect(isPersistError(replay)).toBe(false)
|
||||
if (isPersistError(replay)) throw new Error(replay.error)
|
||||
expect(replay).toEqual(first)
|
||||
expect(await readFile(first.filepath, 'utf8')).toBe(content)
|
||||
})
|
||||
|
||||
test('keeps serialized text blocks complete while exposing an honest JSON preview', async () => {
|
||||
const blocks = [
|
||||
{
|
||||
type: 'text' as const,
|
||||
text: `JSON_HEAD\n${'🙂 structured output\n'.repeat(200)}JSON_TAIL`,
|
||||
},
|
||||
]
|
||||
const serialized = jsonStringify(blocks, null, 2)
|
||||
const result = await persistToolResult(blocks, 'json-preview')
|
||||
expect(isPersistError(result)).toBe(false)
|
||||
if (isPersistError(result)) throw new Error(result.error)
|
||||
|
||||
expect(await readFile(result.filepath, 'utf8')).toBe(serialized)
|
||||
expect(result.originalSize).toBe(byteLength(serialized))
|
||||
expect(result.isJson).toBe(true)
|
||||
expect(result.strategy).toBe('head-only')
|
||||
expect(result.preview).not.toContain('JSON_TAIL')
|
||||
expectWithinBudget(result.preview, PREVIEW_SIZE_BYTES)
|
||||
|
||||
const message = buildLargeToolResultMessage(result)
|
||||
expect(message).toContain(
|
||||
`Output size: ${result.originalSize.toLocaleString('en-US')} bytes (${formatFileSize(result.originalSize)})`,
|
||||
)
|
||||
expect(message).toContain(`Full output saved to: ${result.filepath}`)
|
||||
expect(message).toContain(
|
||||
'UTF-8-safe head-only partial serialized JSON fragment',
|
||||
)
|
||||
expect(message).toContain('may not be valid JSON')
|
||||
expect(message).toContain('2,000-byte total budget')
|
||||
expect(message).not.toContain('Preview (first')
|
||||
})
|
||||
|
||||
test('replays a stored replacement record byte-identically without recomputing it', () => {
|
||||
const legacyReplacement =
|
||||
'<persisted-output>\nLegacy Preview (first 1.9KB)\n界🙂\n</persisted-output>'
|
||||
const message = createUserMessage({
|
||||
content: [
|
||||
{
|
||||
type: 'tool_result',
|
||||
tool_use_id: 'stored-tool-result',
|
||||
content: 'original content that must not trigger a new preview',
|
||||
is_error: false,
|
||||
},
|
||||
],
|
||||
})
|
||||
const state = reconstructContentReplacementState([message], [
|
||||
{
|
||||
kind: 'tool-result',
|
||||
toolUseId: 'stored-tool-result',
|
||||
replacement: legacyReplacement,
|
||||
},
|
||||
])
|
||||
|
||||
const hydrated = applyToolResultReplacementsToMessages(
|
||||
[message],
|
||||
state.replacements,
|
||||
)
|
||||
|
||||
expect(
|
||||
(hydrated[0]!.message.content as Array<{ content: string }>)[0]!.content,
|
||||
).toBe(legacyReplacement)
|
||||
})
|
||||
})
|
||||
|
||||
test('MCP saved-output instructions label the persisted UTF-8 size as bytes', () => {
|
||||
const message = getLargeOutputInstructions(
|
||||
'/tmp/tool-results/mcp-result.txt',
|
||||
12_345,
|
||||
'Plain text',
|
||||
)
|
||||
|
||||
expect(message).toContain('result (12,345 bytes)')
|
||||
expect(message).not.toContain('12,345 characters')
|
||||
})
|
||||
|
||||
test('MCP persistence-failure instructions also use UTF-8 bytes', () => {
|
||||
const message = getLargeOutputPersistenceFailureInstructions(
|
||||
'界🙂',
|
||||
'disk unavailable',
|
||||
)
|
||||
|
||||
expect(message).toContain('result (7 bytes)')
|
||||
expect(message).not.toContain('2 characters')
|
||||
expect(message).toContain('disk unavailable')
|
||||
})
|
||||
@@ -13,11 +13,18 @@ const baseResult = {
|
||||
isJson: false,
|
||||
preview: 'first chunk',
|
||||
hasMore: true,
|
||||
strategy: 'head-tail' as const,
|
||||
}
|
||||
|
||||
test('buildLargeToolResultMessage says "Full output" when the file is complete', () => {
|
||||
const message = buildLargeToolResultMessage(baseResult)
|
||||
expect(message).toContain('Full output saved to: /tmp/tool-results/abc.txt')
|
||||
expect(message).toContain('Output size: 100,000 bytes (97.7KB)')
|
||||
expect(message).toContain(
|
||||
'UTF-8-safe head and tail with an exact omitted-byte marker',
|
||||
)
|
||||
expect(message).toContain('2,000-byte total budget')
|
||||
expect(message).not.toContain('Preview (first')
|
||||
expect(message).not.toContain('capped')
|
||||
})
|
||||
|
||||
|
||||
+297
-26
@@ -3,7 +3,8 @@
|
||||
*/
|
||||
|
||||
import type { ToolResultBlockParam } from '@anthropic-ai/sdk/resources/index.mjs'
|
||||
import { mkdir, writeFile } from 'fs/promises'
|
||||
import { isUtf8 } from 'node:buffer'
|
||||
import { mkdir, open, writeFile, type FileHandle } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { getOriginalCwd, getSessionId } from '../bootstrap/state.js'
|
||||
import {
|
||||
@@ -80,10 +81,12 @@ export function getPersistenceThreshold(
|
||||
// Result of persisting a tool result to disk
|
||||
export type PersistedToolResult = {
|
||||
filepath: string
|
||||
// UTF-8 byte size of the serialized content written to disk.
|
||||
originalSize: number
|
||||
isJson: boolean
|
||||
preview: string
|
||||
hasMore: boolean
|
||||
strategy: PreviewStrategy
|
||||
// When true, the persisted file is capped (only the first portion of the
|
||||
// originalSize-byte output was written). The model-facing message must not
|
||||
// claim the full output is available.
|
||||
@@ -112,6 +115,21 @@ export function getToolResultsDir(): string {
|
||||
// Preview size in bytes for the reference message
|
||||
export const PREVIEW_SIZE_BYTES = 2000
|
||||
|
||||
export type PreviewMode = 'text' | 'json' | 'head-only'
|
||||
export type PreviewStrategy = 'complete' | 'head-tail' | 'head-only'
|
||||
|
||||
export type PreviewResult = {
|
||||
preview: string
|
||||
hasMore: boolean
|
||||
strategy: PreviewStrategy
|
||||
/** Raw source bytes excluded by a generated head/tail marker. */
|
||||
omittedBytes?: number
|
||||
/** UTF-16 offset of the generated marker within preview. */
|
||||
markerStart?: number
|
||||
/** False when retained raw file bytes required replacement during decoding. */
|
||||
retainedBytesValidUtf8?: false
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the filepath where a tool result would be persisted.
|
||||
*/
|
||||
@@ -157,6 +175,7 @@ export async function persistToolResult(
|
||||
await ensureToolResultsDir()
|
||||
const filepath = getToolResultPath(toolUseId, isJson)
|
||||
const contentStr = isJson ? jsonStringify(content, null, 2) : content
|
||||
const originalSize = Buffer.byteLength(contentStr, 'utf8')
|
||||
|
||||
// tool_use_id is unique per invocation and content is deterministic for a
|
||||
// given id, so skip if the file already exists. This prevents re-writing
|
||||
@@ -165,7 +184,7 @@ export async function persistToolResult(
|
||||
try {
|
||||
await writeFile(filepath, contentStr, { encoding: 'utf-8', flag: 'wx' })
|
||||
logForDebugging(
|
||||
`Persisted tool result to ${filepath} (${formatFileSize(contentStr.length)})`,
|
||||
`Persisted tool result to ${filepath} (${formatFileSize(originalSize)})`,
|
||||
)
|
||||
} catch (error) {
|
||||
if (getErrnoCode(error) !== 'EEXIST') {
|
||||
@@ -176,20 +195,37 @@ export async function persistToolResult(
|
||||
}
|
||||
|
||||
// Generate a preview
|
||||
const { preview, hasMore } = generatePreview(contentStr, PREVIEW_SIZE_BYTES)
|
||||
const { preview, hasMore, strategy } = generatePreview(
|
||||
contentStr,
|
||||
PREVIEW_SIZE_BYTES,
|
||||
isJson ? 'json' : 'text',
|
||||
)
|
||||
|
||||
return {
|
||||
filepath,
|
||||
originalSize: contentStr.length,
|
||||
originalSize,
|
||||
isJson,
|
||||
preview,
|
||||
hasMore,
|
||||
strategy,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a message for large tool results with preview
|
||||
*/
|
||||
function describePreviewStrategy(result: PersistedToolResult): string {
|
||||
if (result.strategy === 'head-tail') {
|
||||
return 'UTF-8-safe head and tail with an exact omitted-byte marker'
|
||||
}
|
||||
if (result.strategy === 'head-only') {
|
||||
return result.isJson
|
||||
? 'UTF-8-safe head-only partial serialized JSON fragment (may not be valid JSON)'
|
||||
: 'UTF-8-safe head-only partial output'
|
||||
}
|
||||
return 'complete available inline output'
|
||||
}
|
||||
|
||||
export function buildLargeToolResultMessage(
|
||||
result: PersistedToolResult,
|
||||
): string {
|
||||
@@ -197,10 +233,15 @@ export function buildLargeToolResultMessage(
|
||||
const savedDescription = result.truncated
|
||||
? `Partial output saved to: ${result.filepath} (output was capped — the tail was not saved)`
|
||||
: `Full output saved to: ${result.filepath}`
|
||||
message += `Output too large (${formatFileSize(result.originalSize)}). ${savedDescription}\n\n`
|
||||
message += `Preview (first ${formatFileSize(PREVIEW_SIZE_BYTES)}):\n`
|
||||
const previewDescription = describePreviewStrategy(result)
|
||||
message +=
|
||||
`Output size: ${result.originalSize.toLocaleString('en-US')} bytes ` +
|
||||
`(${formatFileSize(result.originalSize)}). ${savedDescription}\n`
|
||||
message +=
|
||||
`Preview: ${previewDescription}, within a ` +
|
||||
`${PREVIEW_SIZE_BYTES.toLocaleString('en-US')}-byte total budget:\n`
|
||||
message += result.preview
|
||||
message += result.hasMore ? '\n...\n' : '\n'
|
||||
message += '\n'
|
||||
message += PERSISTED_OUTPUT_CLOSING_TAG
|
||||
return message
|
||||
}
|
||||
@@ -331,35 +372,265 @@ async function maybePersistLargeToolResult(
|
||||
logEvent('tengu_tool_result_persisted', {
|
||||
toolName: sanitizeToolNameForAnalytics(toolName),
|
||||
originalSizeBytes: result.originalSize,
|
||||
persistedSizeBytes: message.length,
|
||||
persistedSizeBytes: Buffer.byteLength(message, 'utf8'),
|
||||
estimatedOriginalTokens: Math.ceil(result.originalSize / BYTES_PER_TOKEN),
|
||||
estimatedPersistedTokens: Math.ceil(message.length / BYTES_PER_TOKEN),
|
||||
estimatedPersistedTokens: Math.ceil(
|
||||
Buffer.byteLength(message, 'utf8') / BYTES_PER_TOKEN,
|
||||
),
|
||||
thresholdUsed: threshold,
|
||||
})
|
||||
|
||||
return { ...toolResultBlock, content: message }
|
||||
}
|
||||
|
||||
function safeHeadEnd(buffer: Buffer, maxBytes: number): number {
|
||||
let end = Math.min(Math.max(0, maxBytes), buffer.length)
|
||||
if (end === buffer.length) return end
|
||||
while (end > 0 && (buffer[end]! & 0xc0) === 0x80) end--
|
||||
return end
|
||||
}
|
||||
|
||||
function safeTailStart(buffer: Buffer, maxBytes: number): number {
|
||||
let start = Math.max(0, buffer.length - Math.max(0, maxBytes))
|
||||
while (start < buffer.length && (buffer[start]! & 0xc0) === 0x80) start++
|
||||
return start
|
||||
}
|
||||
|
||||
function decodedByteLength(buffer: Buffer): number {
|
||||
return Buffer.byteLength(buffer.toString('utf8'), 'utf8')
|
||||
}
|
||||
|
||||
function invalidRetainedUtf8Metadata(
|
||||
...buffers: Buffer[]
|
||||
): Pick<PreviewResult, 'retainedBytesValidUtf8'> {
|
||||
return buffers.every(buffer => isUtf8(buffer))
|
||||
? {}
|
||||
: { retainedBytesValidUtf8: false }
|
||||
}
|
||||
|
||||
function fitHeadEnd(buffer: Buffer, targetBytes: number): number {
|
||||
let end = safeHeadEnd(buffer, Math.min(buffer.length, targetBytes))
|
||||
while (
|
||||
end > 0 &&
|
||||
decodedByteLength(buffer.subarray(0, end)) > targetBytes
|
||||
) {
|
||||
end = safeHeadEnd(buffer, end - 1)
|
||||
}
|
||||
return end
|
||||
}
|
||||
|
||||
function fitTailStart(buffer: Buffer, targetBytes: number): number {
|
||||
let start = safeTailStart(buffer, Math.min(buffer.length, targetBytes))
|
||||
while (
|
||||
start < buffer.length &&
|
||||
decodedByteLength(buffer.subarray(start)) > targetBytes
|
||||
) {
|
||||
start = safeTailStart(buffer, buffer.length - start - 1)
|
||||
}
|
||||
return start
|
||||
}
|
||||
|
||||
function chooseHeadEnd(buffer: Buffer, targetBytes: number): number {
|
||||
const hardEnd = fitHeadEnd(buffer, targetBytes)
|
||||
if (hardEnd === 0) return 0
|
||||
const newline = buffer.lastIndexOf(0x0a, hardEnd - 1)
|
||||
const lineEnd = newline + 1
|
||||
const lineBytes = decodedByteLength(buffer.subarray(0, lineEnd))
|
||||
return lineBytes >= targetBytes * 0.5 ? lineEnd : hardEnd
|
||||
}
|
||||
|
||||
function chooseTailStart(buffer: Buffer, targetBytes: number): number {
|
||||
if (targetBytes <= 0) return buffer.length
|
||||
const hardStart = fitTailStart(buffer, targetBytes)
|
||||
const newline = buffer.indexOf(0x0a, hardStart)
|
||||
if (newline === -1) return hardStart
|
||||
const lineStart = newline > hardStart && buffer[newline - 1] === 0x0d
|
||||
? newline - 1
|
||||
: newline
|
||||
const lineBytes = decodedByteLength(buffer.subarray(lineStart))
|
||||
return lineBytes >= targetBytes * 0.5 ? lineStart : hardStart
|
||||
}
|
||||
|
||||
export function formatOmissionMarker(omittedBytes: number): string {
|
||||
return `… ${omittedBytes} bytes omitted …`
|
||||
}
|
||||
|
||||
function getHeadTailTargets(
|
||||
totalBytes: number,
|
||||
maxBytes: number,
|
||||
): { head: number; tail: number } | null {
|
||||
const reservedMarkerBytes = Buffer.byteLength(
|
||||
formatOmissionMarker(totalBytes),
|
||||
'utf8',
|
||||
)
|
||||
if (reservedMarkerBytes > maxBytes) return null
|
||||
const availableBytes = maxBytes - reservedMarkerBytes
|
||||
const head = Math.floor(availableBytes * 0.6)
|
||||
return { head, tail: availableBytes - head }
|
||||
}
|
||||
|
||||
function generateHeadTailPreview(
|
||||
buffer: Buffer,
|
||||
maxBytes: number,
|
||||
): PreviewResult {
|
||||
// Reserve using the total size, whose digit count is at least as wide as
|
||||
// the exact omitted count. The final marker may be a byte or two shorter,
|
||||
// but can never make the preview exceed the budget.
|
||||
const targets = getHeadTailTargets(buffer.length, maxBytes)
|
||||
if (targets === null) {
|
||||
const end = chooseHeadEnd(buffer, maxBytes)
|
||||
const retained = buffer.subarray(0, end)
|
||||
return {
|
||||
preview: retained.toString('utf8'),
|
||||
hasMore: true,
|
||||
strategy: 'head-only',
|
||||
...invalidRetainedUtf8Metadata(retained),
|
||||
}
|
||||
}
|
||||
|
||||
const headEnd = chooseHeadEnd(buffer, targets.head)
|
||||
const tailStart = chooseTailStart(buffer, targets.tail)
|
||||
const omittedBytes = tailStart - headEnd
|
||||
const exactMarker = formatOmissionMarker(omittedBytes)
|
||||
const headBuffer = buffer.subarray(0, headEnd)
|
||||
const tailBuffer = buffer.subarray(tailStart)
|
||||
const head = headBuffer.toString('utf8')
|
||||
return {
|
||||
preview:
|
||||
head +
|
||||
exactMarker +
|
||||
tailBuffer.toString('utf8'),
|
||||
hasMore: true,
|
||||
strategy: 'head-tail',
|
||||
omittedBytes,
|
||||
markerStart: head.length,
|
||||
...invalidRetainedUtf8Metadata(headBuffer, tailBuffer),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a preview of content, truncating at a newline boundary when possible.
|
||||
* Generate a UTF-8 byte-bounded preview. Plain text preserves head and tail;
|
||||
* serialized JSON stays head-only so the preview never resembles valid JSON
|
||||
* assembled from arbitrary fragments.
|
||||
*/
|
||||
export function generatePreview(
|
||||
content: string,
|
||||
maxBytes: number,
|
||||
): { preview: string; hasMore: boolean } {
|
||||
if (content.length <= maxBytes) {
|
||||
return { preview: content, hasMore: false }
|
||||
mode: PreviewMode = 'text',
|
||||
): PreviewResult {
|
||||
const byteLimit = Math.max(0, Math.floor(maxBytes))
|
||||
const contentBytes = Buffer.byteLength(content, 'utf8')
|
||||
if (contentBytes <= byteLimit) {
|
||||
return { preview: content, hasMore: false, strategy: 'complete' }
|
||||
}
|
||||
|
||||
// Find the last newline within the limit to avoid cutting mid-line
|
||||
const truncated = content.slice(0, maxBytes)
|
||||
const lastNewline = truncated.lastIndexOf('\n')
|
||||
const buffer = Buffer.from(content, 'utf8')
|
||||
if (mode !== 'text') {
|
||||
const end = chooseHeadEnd(buffer, byteLimit)
|
||||
return {
|
||||
preview: buffer.subarray(0, end).toString('utf8'),
|
||||
hasMore: true,
|
||||
strategy: 'head-only',
|
||||
}
|
||||
}
|
||||
return generateHeadTailPreview(buffer, byteLimit)
|
||||
}
|
||||
|
||||
// If we found a newline reasonably close to the limit, use it
|
||||
// Otherwise fall back to the exact limit
|
||||
const cutPoint = lastNewline > maxBytes * 0.5 ? lastNewline : maxBytes
|
||||
async function readFileBytes(
|
||||
handle: FileHandle,
|
||||
position: number,
|
||||
length: number,
|
||||
): Promise<Buffer> {
|
||||
const buffer = Buffer.alloc(length)
|
||||
let offset = 0
|
||||
while (offset < length) {
|
||||
const { bytesRead } = await handle.read(
|
||||
buffer,
|
||||
offset,
|
||||
length - offset,
|
||||
position + offset,
|
||||
)
|
||||
if (bytesRead === 0) break
|
||||
offset += bytesRead
|
||||
}
|
||||
return buffer.subarray(0, offset)
|
||||
}
|
||||
|
||||
return { preview: content.slice(0, cutPoint), hasMore: true }
|
||||
/**
|
||||
* Build the same bounded text preview from a file without loading the full
|
||||
* spill into memory. Size is derived from the opened handle so callers cannot
|
||||
* provide stale metadata that would make the omitted-byte marker inaccurate.
|
||||
*/
|
||||
export async function generateFilePreview(
|
||||
filepath: string,
|
||||
maxBytes: number,
|
||||
): Promise<PreviewResult> {
|
||||
const byteLimit = Math.max(0, Math.floor(maxBytes))
|
||||
const handle = await open(filepath, 'r')
|
||||
try {
|
||||
const fileSizeBytes = (await handle.stat()).size
|
||||
if (fileSizeBytes <= byteLimit) {
|
||||
const content = await readFileBytes(handle, 0, fileSizeBytes)
|
||||
const decoded = content.toString('utf8')
|
||||
if (Buffer.byteLength(decoded, 'utf8') > byteLimit) {
|
||||
return generateHeadTailPreview(content, byteLimit)
|
||||
}
|
||||
return {
|
||||
preview: decoded,
|
||||
hasMore: false,
|
||||
strategy: 'complete',
|
||||
...invalidRetainedUtf8Metadata(content),
|
||||
}
|
||||
}
|
||||
|
||||
const targets = getHeadTailTargets(fileSizeBytes, byteLimit)
|
||||
if (targets === null) {
|
||||
const head = await readFileBytes(
|
||||
handle,
|
||||
0,
|
||||
Math.min(fileSizeBytes, byteLimit + 3),
|
||||
)
|
||||
const headEnd = chooseHeadEnd(head, byteLimit)
|
||||
const retained = head.subarray(0, headEnd)
|
||||
return {
|
||||
preview: retained.toString('utf8'),
|
||||
hasMore: true,
|
||||
strategy: 'head-only',
|
||||
...invalidRetainedUtf8Metadata(retained),
|
||||
}
|
||||
}
|
||||
|
||||
// Up to three lookahead bytes are enough to detect whether a requested
|
||||
// boundary falls inside a four-byte UTF-8 code point.
|
||||
const headReadLength = Math.min(fileSizeBytes, targets.head + 3)
|
||||
const tailReadLength = Math.min(fileSizeBytes, targets.tail + 3)
|
||||
const tailReadStart = fileSizeBytes - tailReadLength
|
||||
const [head, tail] = await Promise.all([
|
||||
readFileBytes(handle, 0, headReadLength),
|
||||
readFileBytes(handle, tailReadStart, tailReadLength),
|
||||
])
|
||||
const headEnd = chooseHeadEnd(head, targets.head)
|
||||
const localTailStart = chooseTailStart(tail, targets.tail)
|
||||
const tailStart = tailReadStart + localTailStart
|
||||
const omittedBytes = tailStart - headEnd
|
||||
const marker = formatOmissionMarker(omittedBytes)
|
||||
const headBuffer = head.subarray(0, headEnd)
|
||||
const tailBuffer = tail.subarray(localTailStart)
|
||||
const headPreview = headBuffer.toString('utf8')
|
||||
return {
|
||||
preview:
|
||||
headPreview +
|
||||
marker +
|
||||
tailBuffer.toString('utf8'),
|
||||
hasMore: true,
|
||||
strategy: 'head-tail',
|
||||
omittedBytes,
|
||||
markerStart: headPreview.length,
|
||||
...invalidRetainedUtf8Metadata(headBuffer, tailBuffer),
|
||||
}
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -888,7 +1159,7 @@ export async function enforceToolResultBudget(
|
||||
toPersist.map(async c => [c, await buildReplacement(c)] as const),
|
||||
)
|
||||
const newlyReplaced: ToolResultReplacementRecord[] = []
|
||||
let replacedSize = 0
|
||||
let replacedSizeBytes = 0
|
||||
for (const [candidate, replacement] of freshReplacements) {
|
||||
// Mark seen HERE, post-await, atomically with replacements.set for
|
||||
// success cases. For persist failures (replacement === null) the ID
|
||||
@@ -896,7 +1167,7 @@ export async function enforceToolResultBudget(
|
||||
// model, so treating it as frozen going forward is correct.
|
||||
state.seenIds.add(candidate.toolUseId)
|
||||
if (replacement === null) continue
|
||||
replacedSize += candidate.size
|
||||
replacedSizeBytes += replacement.originalSize
|
||||
replacementMap.set(candidate.toolUseId, replacement.content)
|
||||
state.replacements.set(candidate.toolUseId, replacement.content)
|
||||
newlyReplaced.push({
|
||||
@@ -906,12 +1177,12 @@ export async function enforceToolResultBudget(
|
||||
})
|
||||
logEvent('tengu_tool_result_persisted_message_budget', {
|
||||
originalSizeBytes: replacement.originalSize,
|
||||
persistedSizeBytes: replacement.content.length,
|
||||
persistedSizeBytes: Buffer.byteLength(replacement.content, 'utf8'),
|
||||
estimatedOriginalTokens: Math.ceil(
|
||||
replacement.originalSize / BYTES_PER_TOKEN,
|
||||
),
|
||||
estimatedPersistedTokens: Math.ceil(
|
||||
replacement.content.length / BYTES_PER_TOKEN,
|
||||
Buffer.byteLength(replacement.content, 'utf8') / BYTES_PER_TOKEN,
|
||||
),
|
||||
})
|
||||
}
|
||||
@@ -924,12 +1195,12 @@ export async function enforceToolResultBudget(
|
||||
logForDebugging(
|
||||
`Per-message budget: persisted ${newlyReplaced.length} tool results ` +
|
||||
`across ${messagesOverBudget} over-budget message(s), ` +
|
||||
`shed ~${formatFileSize(replacedSize)}, ${reappliedCount} re-applied`,
|
||||
`shed ~${formatFileSize(replacedSizeBytes)}, ${reappliedCount} re-applied`,
|
||||
)
|
||||
logEvent('tengu_message_level_tool_result_budget_enforced', {
|
||||
resultsPersisted: newlyReplaced.length,
|
||||
messagesOverBudget,
|
||||
replacedSizeBytes: replacedSize,
|
||||
replacedSizeBytes,
|
||||
reapplied: reappliedCount,
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user