diff --git a/src/hooks/useTextInput.test.ts b/src/hooks/useTextInput.test.ts index 19cd1339c..e5e413d32 100644 --- a/src/hooks/useTextInput.test.ts +++ b/src/hooks/useTextInput.test.ts @@ -1,10 +1,19 @@ -import { describe, expect, test } from 'bun:test' +import { PassThrough } from 'node:stream' +import { describe, expect, test } from 'bun:test' +import { createElement, useState } from 'react' + +import { createRoot, type Key } from '../ink.js' +import { AppStateProvider } from '../state/AppState.js' +import type { TextInputState } from '../types/textInputTypes.js' import { Cursor } from '../utils/Cursor.js' import { applyCoalescedDelInput, applyPrintableInput, + composeCombiningMark, prepareTextInputEvent, + replacePreviousWithChar, + useTextInput, } from './useTextInput.js' const insert = (cursor: Cursor, text: string): Cursor => cursor.insert(text) @@ -23,6 +32,82 @@ test('applyPrintableInput detects an ANSI-wrapped mode character', () => { expect(notifications).toEqual(['!']) }) +test('applyPrintableInput inserts NFD input fully composed', () => { + const result = applyPrintableInput(Cursor.fromText('', 80, 0), 'a\u0306') + + expect(result?.text).toBe('ă') + expect(result?.offset).toBe(1) +}) + +describe('composeCombiningMark', () => { + test('composes a standalone breve onto the preceding vowel', () => { + expect(composeCombiningMark('a', 1, '\u0306')).toEqual({ + text: 'ă', + offset: 1, + }) + }) + + test('composes sequential IME marks into a single precomposed char', () => { + // tiếng: e + circumflex → ê, then acute → ế (U+1EBF, one code unit) + const stepOne = composeCombiningMark('tie', 3, '\u0302') + expect(stepOne?.text).toBe('tiê') + + const stepTwo = composeCombiningMark(stepOne!.text, stepOne!.offset, '\u0301') + expect(stepTwo?.text).toBe('tiế') + expect([...(stepTwo?.text ?? '')].length).toBe(4) + }) + + test('composes mid-text without disturbing trailing characters', () => { + // Offset 1 = cursor right after the base vowel "a", mirroring real NFD + // arrival: the mark composes onto the character before the cursor. + expect(composeCombiningMark('ab c', 1, '\u0306')).toEqual({ + text: 'ăb c', + offset: 1, + }) + }) + + test('composes a mark outside the old U+0300-U+036F range (Hebrew point)', () => { + // HEBREW POINT SHEVA (U+05B0, general category Mn) sits outside the + // previous [\u0300-\u036f] matcher; \p{M} must still compose it. + expect(composeCombiningMark('בית', 1, '\u05B0')).toEqual({ + text: 'ב\u05B0ית', + offset: 2, + }) + }) + + test('returns null when nothing precedes the cursor', () => { + expect(composeCombiningMark('', 0, '\u0306')).toBeNull() + }) + + test('returns null for non-mark input', () => { + expect(composeCombiningMark('a', 1, 'w')).toBeNull() + }) +}) + +describe('replacePreviousWithChar', () => { + test('replaces the previous character with the composed replacement', () => { + expect(replacePreviousWithChar('xin cha', 7, 'ò')).toEqual({ + text: 'xin chà', + offset: 7, + }) + }) + + test('replaces mid-text preserving surrounding characters', () => { + expect(replacePreviousWithChar('abc', 2, 'ă')).toEqual({ + text: 'aăc', + offset: 2, + }) + }) + + test('returns null when nothing precedes the cursor', () => { + expect(replacePreviousWithChar('a', 0, 'ă')).toBeNull() + }) + + test('ignores ASCII replacements', () => { + expect(replacePreviousWithChar('a', 1, 'b')).toBeNull() + }) +}) + function apply( text: string, input: string, @@ -40,6 +125,14 @@ describe('applyCoalescedDelInput', () => { expect(apply('abc', '\x7f').cursor.text).toBe('ab') }) + test('treats Ctrl-H backspace bytes like DEL', () => { + expect(apply('a', '\bă').cursor.text).toBe('ă') + }) + + test('handles mixed DEL and Ctrl-H runs before inserting', () => { + expect(apply('abc', '\x7f\bă').cursor.text).toBe('aă') + }) + test('inserts replacement text after DEL', () => { expect(apply('a', '\x7fă').cursor.text).toBe('ă') }) @@ -243,3 +336,104 @@ describe('prepareTextInputEvent', () => { }) }) }) + +async function waitFor( + predicate: () => boolean, + timeoutMs = 2500, +): Promise { + const startedAt = Date.now() + + while (Date.now() - startedAt < timeoutMs) { + if (predicate()) return + await Bun.sleep(5) + } + + throw new Error('Timed out waiting for useTextInput state') +} + +// Renders useTextInput through its public onInput API (same probe pattern +// as components/TextInput.test.tsx) so IME composition paths can be +// exercised end-to-end against user-visible text and cursor outcomes. +async function runOnInputScenario(options: { + initialValue: string + input: string + key?: Partial +}): Promise<{ value: string; cursorOffset: number }> { + const { initialValue, input } = options + const key = options.key ?? {} + let inputState: TextInputState | undefined + let observedValue = initialValue + let observedCursorOffset = initialValue.length + + function ImeProbe(): null { + const [value, setValue] = useState(initialValue) + const [offset, setOffset] = useState(initialValue.length) + + inputState = useTextInput({ + value, + onChange: nextValue => { + observedValue = nextValue + setValue(nextValue) + }, + onSubmit: () => {}, + cursorChar: ' ', + invert: text => text, + themeText: text => text, + columns: 60, + externalOffset: offset, + onOffsetChange: nextOffset => { + observedCursorOffset = nextOffset + setOffset(nextOffset) + }, + multiline: true, + }) + + return null + } + + const stdout = new PassThrough() + ;(stdout as unknown as { columns: number }).columns = 80 + const root = await createRoot({ + stdout: stdout as unknown as NodeJS.WriteStream, + patchConsole: false, + }) + + try { + root.render(createElement(AppStateProvider, null, createElement(ImeProbe))) + await waitFor(() => inputState !== undefined) + inputState!.onInput(input, key as Key) + await Bun.sleep(25) + } finally { + root.unmount() + } + + return { value: observedValue, cursorOffset: observedCursorOffset } +} + +describe('useTextInput IME composition regression (#2018)', () => { + test('composes a backspace-flagged replacement into user-visible text', async () => { + // Telex/VNI compose events arrive flagged as backspace while carrying + // the precomposed replacement character; the visible result is the + // composed word, not a deleted character plus stray text. + const result = await runOnInputScenario({ + initialValue: 'xin cha', + input: 'ò', + key: { backspace: true }, + }) + + expect(result.value).toBe('xin chà') + expect(result.cursorOffset).toBe(7) + }) + + test('composes a delayed standalone combining mark onto its base character', async () => { + // NFD path: the base vowel commits first and the tone mark arrives + // later as its own standalone text event. + const result = await runOnInputScenario({ + initialValue: 'tiê', + input: '\u0301', + }) + + expect(result.value).toBe('tiế') + expect(result.cursorOffset).toBe(3) + }) +}) diff --git a/src/hooks/useTextInput.ts b/src/hooks/useTextInput.ts index 57a1ae063..0a920ed23 100644 --- a/src/hooks/useTextInput.ts +++ b/src/hooks/useTextInput.ts @@ -37,6 +37,64 @@ function mapInput(input_map: Array<[string, InputHandler]>): InputMapper { } } +// Any Unicode combining mark (general category M) — covers Vietnamese +// tone/breath marks plus marks outside the U+0300-U+036F block (Hebrew +// points, Devanagari matras, ...) when an IME emits NFD instead of +// precomposed forms. +const COMBINING_MARK_RE = /^\p{M}/u + +export type ComposedTextEdit = { + text: string + offset: number +} + +/** + * Compose a standalone combining mark (NFD input) onto the character before + * the cursor. Returns null when the input is not a combining mark or there + * is nothing to compose with (#2018). + */ +export function composeCombiningMark( + text: string, + offset: number, + input: string, +): ComposedTextEdit | null { + if (!COMBINING_MARK_RE.test(input) || offset <= 0 || offset > text.length) { + return null + } + const newBefore = (text.slice(0, offset) + input).normalize('NFC') + return { + text: newBefore + text.slice(offset), + offset: newBefore.length, + } +} + +/** + * Telex/VNI compose pattern: some IMEs/terminals deliver composition as a + * backspace-flagged event carrying the precomposed replacement character. + * Replace the previous character instead of treating the pair as delete + * plus literal insert. Returns null when the input is not a printable + * non-ASCII replacement (#2018). + */ +export function replacePreviousWithChar( + text: string, + offset: number, + input: string, +): ComposedTextEdit | null { + if ( + input.length !== 1 || + (input.codePointAt(0) ?? 0) <= 127 || + offset <= 0 || + offset > text.length + ) { + return null + } + const newBefore = text.slice(0, offset - 1) + input + return { + text: (newBefore + text.slice(offset)).normalize('NFC'), + offset: newBefore.normalize('NFC').length, + } +} + export function prepareTextInputEvent(input: string): { input: string shouldSubmit: boolean @@ -84,7 +142,10 @@ export function applyPrintableInput( return cursor.endOfLine() } - const text = stripAnsi(input) + // Normalize to NFC so decomposed IME input composes into exactly what + // the cursor's MeasuredText stores, keeping returned offsets aligned + // when normalization shrinks code-unit length (#2018). + const text = stripAnsi(input).normalize('NFC') if ( !options.modeCharacterIsText && cursor.text.length === 0 && @@ -112,7 +173,9 @@ export function applyCoalescedDelInput( let shouldCommit = true for (let index = 0; index < input.length; index++) { - if (input[index] !== '\x7f') continue + // Both DEL bytes arrive from IMEs/terminals: \x7f is the common + // backspace, \b (Ctrl-H) is emitted by some Vietnamese IME setups. + if (input[index] !== '\x7f' && input[index] !== '\b') continue if (index > segmentStart) { const insertedCursor = insert( @@ -128,7 +191,10 @@ export function applyCoalescedDelInput( } let delEnd = index + 1 - while (delEnd < input.length && input[delEnd] === '\x7f') { + while ( + delEnd < input.length && + (input[delEnd] === '\x7f' || input[delEnd] === '\b') + ) { delEnd++ } const delCount = delEnd - index @@ -631,14 +697,53 @@ export function useTextInput({ return } + // Vietnamese/CJK IME composition paths (#2018). Both bypass the \r / + // coalesced-Enter handling in prepareTextInputEvent, so they only run + // on inputs that carry no carriage returns. + if (!filteredInput.includes('\r')) { + // Telex/VNI backspace+replacement compose: a backspace-flagged event + // carrying a printable non-ASCII character replaces the previous one + // (a → ă) instead of deleting it. + if (key.backspace && !key.ctrl && !key.meta) { + const replaced = replacePreviousWithChar( + currentCursor.text, + currentCursor.offset, + filteredInput, + ) + if (replaced) { + resetKillAccumulation() + resetYankState() + setValue(replaced.text, replaced.offset) + return + } + } + + // Standalone combining mark (NFD input): compose onto the preceding + // character rather than inserting a bare mark that later renders, + // measures, and deletes as a separate unit. + const markComposed = composeCombiningMark( + currentCursor.text, + currentCursor.offset, + filteredInput, + ) + if (markComposed) { + resetKillAccumulation() + resetYankState() + setValue(markComposed.text, markComposed.offset) + return + } + } + const preparedInput = prepareTextInputEvent(filteredInput) // Fix Issue #1853: Filter DEL characters that interfere with backspace in SSH/tmux - // In SSH/tmux environments, backspace generates both key events and raw DEL chars + // In SSH/tmux environments, backspace generates both key events and raw DEL chars. + // \b (Ctrl-H) is included because Vietnamese IME setups emit it as the + // backside of their Telex/VNI backspace+replacement compose pattern (#2018). if ( !key.backspace && !key.delete && - filteredInput.includes('\x7f') + /[\x7f\b]/.test(filteredInput) ) { let finalChangeContext: TextInputChangeContext | undefined let modeEntryChangeContext: diff --git a/src/ink/components/App.test.tsx b/src/ink/components/App.test.tsx index 30014f450..7e9715746 100644 --- a/src/ink/components/App.test.tsx +++ b/src/ink/components/App.test.tsx @@ -1,9 +1,11 @@ import { EventEmitter } from 'node:events' import { PassThrough } from 'node:stream' -import { afterEach, describe, expect, mock, test } from 'bun:test' +import { afterEach, describe, expect, jest, mock, test } from 'bun:test' +import type { ParsedKey } from '../parse-keypress.js' import { createSelectionState } from '../selection.js' +import { PASTE_START } from '../termio/csi.js' import App from './App.js' type FakeStdin = NodeJS.ReadStream & { @@ -33,7 +35,10 @@ function createFakeStdout(): NodeJS.WriteStream { return stdout } -function createApp(stdin: NodeJS.ReadStream): App { +function createApp( + stdin: NodeJS.ReadStream, + overrides: { dispatchKeyboardEvent?: (parsedKey: ParsedKey) => void } = {}, +): App { return new App({ children: null, stdin, @@ -51,7 +56,7 @@ function createApp(stdin: NodeJS.ReadStream): App { onOpenHyperlink: () => {}, onMultiClick: () => {}, onSelectionDrag: () => {}, - dispatchKeyboardEvent: () => {}, + dispatchKeyboardEvent: overrides.dispatchKeyboardEvent ?? (() => {}), }) } @@ -117,3 +122,100 @@ describe('App stdin mode setup', () => { app.handleSetRawMode(false) }) }) + +describe('App incomplete-sequence flush timers', () => { + afterEach(() => { + jest.useRealTimers() + }) + + function createDispatchCollector(): { + dispatched: ParsedKey[] + dispatchKeyboardEvent: (parsedKey: ParsedKey) => void + } { + const dispatched: ParsedKey[] = [] + return { + dispatched, + dispatchKeyboardEvent: key => { + dispatched.push(key) + }, + } + } + + test('holds a lone Escape, then flushes it exactly at NORMAL_TIMEOUT', () => { + jest.useFakeTimers() + const { dispatched, dispatchKeyboardEvent } = createDispatchCollector() + const app = createApp(createFakeStdin(), { dispatchKeyboardEvent }) + + // The bare ESC is buffered, not emitted as an instant Escape keypress. + app.processInput('\x1b') + expect(dispatched).toEqual([]) + + jest.advanceTimersByTime(app.NORMAL_TIMEOUT - 1) + expect(dispatched).toEqual([]) + + jest.advanceTimersByTime(1) + expect(dispatched).toHaveLength(1) + expect(dispatched[0]?.name).toBe('escape') + }) + + test('holds an Alt-prefixed half and composes it when the rest arrives before the flush', () => { + jest.useFakeTimers() + const { dispatched, dispatchKeyboardEvent } = createDispatchCollector() + const app = createApp(createFakeStdin(), { dispatchKeyboardEvent }) + + app.processInput('\x1b') + expect(dispatched).toEqual([]) + + // Continuation lands inside the hold window: ESC + b composes Alt+b. + app.processInput('b') + expect(dispatched).toHaveLength(1) + expect(dispatched[0]?.sequence).toBe('\x1bb') + expect(dispatched[0]?.meta).toBe(true) + + // The satisfied hold must not leak a phantom Escape when its timer fires. + jest.advanceTimersByTime(app.NORMAL_TIMEOUT) + expect(dispatched).toHaveLength(1) + }) + + test('composes a delayed CSI-u chunk that arrives before the flush', () => { + jest.useFakeTimers() + const { dispatched, dispatchKeyboardEvent } = createDispatchCollector() + const app = createApp(createFakeStdin(), { dispatchKeyboardEvent }) + + app.processInput('\x1b[98') + expect(dispatched).toEqual([]) + + // IME/CSI-u second half arrives within the hold window: parses as + // kitty Alt+b instead of garbage after a premature flush (#2018). + app.processInput(';3u') + expect(dispatched).toHaveLength(1) + expect(dispatched[0]?.name).toBe('b') + expect(dispatched[0]?.meta).toBe(true) + + jest.advanceTimersByTime(app.NORMAL_TIMEOUT) + expect(dispatched).toHaveLength(1) + }) + + test('holds an incomplete bracketed paste past PASTE_TIMEOUT, then flushes it as one paste', () => { + jest.useFakeTimers() + const { dispatched, dispatchKeyboardEvent } = createDispatchCollector() + const app = createApp(createFakeStdin(), { dispatchKeyboardEvent }) + + // Paste start + content + truncated CSI tail: tokenizer stays buffered + // and App stays in paste mode, so nothing is emitted yet. + app.processInput(`${PASTE_START}hello\x1b[2`) + expect(dispatched).toEqual([]) + + jest.advanceTimersByTime(app.PASTE_TIMEOUT - 1) + expect(dispatched).toEqual([]) + + jest.advanceTimersByTime(1) + expect(dispatched).toHaveLength(1) + expect(dispatched[0]?.isPasted).toBe(true) + expect(dispatched[0]?.sequence).toBe('hello\x1b[2') + + // The flush consumed both the paste buffer and the incomplete tail. + expect(app.keyParseState.incomplete).toBe('') + expect(app.keyParseState.mode).toBe('NORMAL') + }) +}) diff --git a/src/ink/components/App.tsx b/src/ink/components/App.tsx index e52e5ddba..d7e7660d4 100644 --- a/src/ink/components/App.tsx +++ b/src/ink/components/App.tsx @@ -121,8 +121,13 @@ export default class App extends PureComponent { // where startup input appears frozen when data mode is the default. stdinMode: 'readable' | 'data' = process.env.OPENCLAUDE_USE_DATA_STDIN === '1' || process.env.OPENCLAUDE_USE_READABLE_STDIN === '0' ? 'data' : 'readable'; // Timeout durations for incomplete sequences (ms) - readonly NORMAL_TIMEOUT = 50; // Short timeout for regular esc sequences - readonly PASTE_TIMEOUT = 500; // Longer timeout for paste operations + // NORMAL_TIMEOUT must exceed IME composition gaps: Vietnamese Telex/VNI + // and CJK IMEs emit multi-byte UTF-8 / CSI-u sequences whose halves can + // arrive with inter-chunk pauses. Flushing at 50ms split those sequences + // into garbage keys (#2018); 300ms gives composition room while keeping + // lone-escape detection acceptable. + readonly NORMAL_TIMEOUT = 300; + readonly PASTE_TIMEOUT = 1000; // Longer timeout for paste operations // Terminal query/response dispatch. Responses arrive on stdin (parsed // out by parse-keypress) and are routed to pending promise resolvers. diff --git a/src/ink/events/input-event.ts b/src/ink/events/input-event.ts index 490502811..8a0696a3e 100644 --- a/src/ink/events/input-event.ts +++ b/src/ink/events/input-event.ts @@ -186,6 +186,23 @@ function parseKey(keypress: ParsedKey): [Key, string] { key.shift = true } + // Printable non-ASCII characters (Vietnamese precomposed letters, CJK, + // accented Latin) arrive as plain text sequences with no keyName-map + // entry, leaving keypress.name empty. Record the character as the name + // so downstream consumers (keyboard-event dispatch, mode handlers) see + // the character instead of an empty string. This runs after the + // nonAlphanumericKeys clear above, and single-code-point names are never + // members of that list, so this cannot cause input to be cleared (#2018). + // Count code points, not UTF-16 units, so one astral code point (e.g. + // emoji) still qualifies. + if ( + !keypress.name && + [...input].length === 1 && + (input.codePointAt(0) ?? 0) > 127 + ) { + keypress.name = input + } + return [key, input] } diff --git a/src/ink/parse-keypress.test.ts b/src/ink/parse-keypress.test.ts index 1ac4be7eb..762e16274 100644 --- a/src/ink/parse-keypress.test.ts +++ b/src/ink/parse-keypress.test.ts @@ -74,3 +74,78 @@ test('preserves Vietnamese UTF-8 input split across stdin chunks', () => { expect(events.map(event => event.input).join('')).toBe('tiếng Việt') expect(events.some(event => event.input.includes('\uFFFD'))).toBe(false) }) + +test('names precomposed Vietnamese characters typed as plain text', () => { + const event = parseInputEvent('ă') + + expect(event.input).toBe('ă') + expect(event.keypress.name).toBe('ă') +}) + +test('composes NFD Vietnamese text into a single precomposed key', () => { + const event = parseInputEvent('a\u0306') + + expect(event.input).toBe('ă') + expect(event.input.codePointAt(0)).toBe(0x0103) + expect(event.keypress.name).toBe('ă') +}) + +test('names standalone combining marks so they reach input handlers', () => { + const event = parseInputEvent('\u0306') + + expect(event.input).toBe('\u0306') + expect(event.keypress.name).toBe('\u0306') +}) + +test('preserves multi-character Vietnamese words as one input event', () => { + const [items] = parseMultipleKeypresses(INITIAL_STATE, 'chào') + + expect(items).toHaveLength(1) + const item = items[0] + expect(item?.kind).toBe('key') + expect((item as ParsedKey).sequence).toBe('chào') +}) + +test('preserves Vietnamese CSI-u input', () => { + const event = parseInputEvent('\x1b[259u') + + expect(event.input).toBe('ă') + expect(event.keypress.name).toBe('ă') +}) + +test('keeps DEL plus replacement intact for downstream coalescing', () => { + const event = parseInputEvent('\x7fă') + + expect(event.input).toBe('\x7fă') + expect(event.key.backspace).toBe(false) +}) + +test('names astral-plane characters typed as plain text', () => { + // 😀 is one code point but two UTF-16 units; the printable non-ASCII + // branch must count code points so the key still gets a name. + const event = parseInputEvent('😀') + + expect(event.input).toBe('😀') + expect(event.keypress.name).toBe('😀') +}) + +test('InputEvent names astral sequences that reach it without a name', () => { + const unnamedAstral: ParsedKey = { + kind: 'key', + fn: false, + name: '', + ctrl: false, + meta: false, + shift: false, + option: false, + super: false, + sequence: '😀', + raw: '😀', + isPasted: false, + } + + const event = new InputEvent(unnamedAstral) + + expect(event.input).toBe('😀') + expect(event.keypress.name).toBe('😀') +}) diff --git a/src/ink/parse-keypress.ts b/src/ink/parse-keypress.ts index 7df7db919..9a67c8800 100644 --- a/src/ink/parse-keypress.ts +++ b/src/ink/parse-keypress.ts @@ -347,7 +347,11 @@ export function parseMultipleKeypresses( const mouse = parseMouseEvent(resynthesized) keys.push(mouse ?? parseKeypress(resynthesized)) } else { - keys.push(parseKeypress(token.value)) + // IMEs and some terminals emit decomposed Unicode (NFD) — e.g. + // U+0061 + U+0306 instead of precomposed U+0103 (ă). Normalize + // typed text to NFC here so a decomposed pair parses as one + // composed character instead of arriving as separate keys (#2018). + keys.push(parseKeypress(token.value.normalize('NFC'))) } } } @@ -830,6 +834,15 @@ function parseKeypress(s: string = ''): ParsedKey { } else if (s.length === 1 && s >= 'A' && s <= 'Z') { key.name = s.toLowerCase() key.shift = true + } else if ([...s].length === 1 && (s.codePointAt(0) ?? 0) > 127) { + // Printable non-ASCII character (accented Latin, CJK, astral emoji, + // etc.). Count code points, not UTF-16 units, so a single astral code + // point (UTF-16 length 2, e.g. 😀) is still recognized. Assign it as the + // key name so downstream consumers (keybindings, vim mode, DOM keyboard + // dispatch) can identify it; text insertion still flows through + // `sequence`. Without this, Vietnamese precomposed characters like + // ă/ơ/ư fall through unnamed (#2018). + key.name = s } else if ((parts = META_KEY_CODE_RE.exec(s))) { key.meta = true key.shift = /^[A-Z]$/.test(parts[1]!)