fix(tui): proper Unicode/IME input handling for composed sequences (#2018) (#2154)

* fix(tui): proper Unicode/IME input handling for composed sequences (#2018)

* test(tui): address CodeRabbit review - hook-level IME coverage, full Unicode marks, astral code points, timeout regression tests
This commit is contained in:
Cal
2026-08-24 10:21:57 +08:00
committed by GitHub
parent 54f963d006
commit e8026263ca
7 changed files with 523 additions and 12 deletions
+195 -1
View File
@@ -1,10 +1,19 @@
import { describe, expect, test } from 'bun:test'
import { PassThrough } from 'node:stream'
import { describe, expect, test } from 'bun:test'
import { createElement, useState } from 'react'
import { createRoot, type Key } from '../ink.js'
import { AppStateProvider } from '../state/AppState.js'
import type { TextInputState } from '../types/textInputTypes.js'
import { Cursor } from '../utils/Cursor.js'
import {
applyCoalescedDelInput,
applyPrintableInput,
composeCombiningMark,
prepareTextInputEvent,
replacePreviousWithChar,
useTextInput,
} from './useTextInput.js'
const insert = (cursor: Cursor, text: string): Cursor => cursor.insert(text)
@@ -23,6 +32,82 @@ test('applyPrintableInput detects an ANSI-wrapped mode character', () => {
expect(notifications).toEqual(['!'])
})
test('applyPrintableInput inserts NFD input fully composed', () => {
const result = applyPrintableInput(Cursor.fromText('', 80, 0), 'a\u0306')
expect(result?.text).toBe('ă')
expect(result?.offset).toBe(1)
})
describe('composeCombiningMark', () => {
test('composes a standalone breve onto the preceding vowel', () => {
expect(composeCombiningMark('a', 1, '\u0306')).toEqual({
text: 'ă',
offset: 1,
})
})
test('composes sequential IME marks into a single precomposed char', () => {
// tiếng: e + circumflex → ê, then acute → ế (U+1EBF, one code unit)
const stepOne = composeCombiningMark('tie', 3, '\u0302')
expect(stepOne?.text).toBe('tiê')
const stepTwo = composeCombiningMark(stepOne!.text, stepOne!.offset, '\u0301')
expect(stepTwo?.text).toBe('tiế')
expect([...(stepTwo?.text ?? '')].length).toBe(4)
})
test('composes mid-text without disturbing trailing characters', () => {
// Offset 1 = cursor right after the base vowel "a", mirroring real NFD
// arrival: the mark composes onto the character before the cursor.
expect(composeCombiningMark('ab c', 1, '\u0306')).toEqual({
text: 'ăb c',
offset: 1,
})
})
test('composes a mark outside the old U+0300-U+036F range (Hebrew point)', () => {
// HEBREW POINT SHEVA (U+05B0, general category Mn) sits outside the
// previous [\u0300-\u036f] matcher; \p{M} must still compose it.
expect(composeCombiningMark('בית', 1, '\u05B0')).toEqual({
text: 'ב\u05B0ית',
offset: 2,
})
})
test('returns null when nothing precedes the cursor', () => {
expect(composeCombiningMark('', 0, '\u0306')).toBeNull()
})
test('returns null for non-mark input', () => {
expect(composeCombiningMark('a', 1, 'w')).toBeNull()
})
})
describe('replacePreviousWithChar', () => {
test('replaces the previous character with the composed replacement', () => {
expect(replacePreviousWithChar('xin cha', 7, 'ò')).toEqual({
text: 'xin chà',
offset: 7,
})
})
test('replaces mid-text preserving surrounding characters', () => {
expect(replacePreviousWithChar('abc', 2, 'ă')).toEqual({
text: 'aăc',
offset: 2,
})
})
test('returns null when nothing precedes the cursor', () => {
expect(replacePreviousWithChar('a', 0, 'ă')).toBeNull()
})
test('ignores ASCII replacements', () => {
expect(replacePreviousWithChar('a', 1, 'b')).toBeNull()
})
})
function apply(
text: string,
input: string,
@@ -40,6 +125,14 @@ describe('applyCoalescedDelInput', () => {
expect(apply('abc', '\x7f').cursor.text).toBe('ab')
})
test('treats Ctrl-H backspace bytes like DEL', () => {
expect(apply('a', '\bă').cursor.text).toBe('ă')
})
test('handles mixed DEL and Ctrl-H runs before inserting', () => {
expect(apply('abc', '\x7f\bă').cursor.text).toBe('aă')
})
test('inserts replacement text after DEL', () => {
expect(apply('a', '\x7fă').cursor.text).toBe('ă')
})
@@ -243,3 +336,104 @@ describe('prepareTextInputEvent', () => {
})
})
})
async function waitFor(
predicate: () => boolean,
timeoutMs = 2500,
): Promise<void> {
const startedAt = Date.now()
while (Date.now() - startedAt < timeoutMs) {
if (predicate()) return
await Bun.sleep(5)
}
throw new Error('Timed out waiting for useTextInput state')
}
// Renders useTextInput through its public onInput API (same probe pattern
// as components/TextInput.test.tsx) so IME composition paths can be
// exercised end-to-end against user-visible text and cursor outcomes.
async function runOnInputScenario(options: {
initialValue: string
input: string
key?: Partial<Key>
}): Promise<{ value: string; cursorOffset: number }> {
const { initialValue, input } = options
const key = options.key ?? {}
let inputState: TextInputState | undefined
let observedValue = initialValue
let observedCursorOffset = initialValue.length
function ImeProbe(): null {
const [value, setValue] = useState(initialValue)
const [offset, setOffset] = useState(initialValue.length)
inputState = useTextInput({
value,
onChange: nextValue => {
observedValue = nextValue
setValue(nextValue)
},
onSubmit: () => {},
cursorChar: ' ',
invert: text => text,
themeText: text => text,
columns: 60,
externalOffset: offset,
onOffsetChange: nextOffset => {
observedCursorOffset = nextOffset
setOffset(nextOffset)
},
multiline: true,
})
return null
}
const stdout = new PassThrough()
;(stdout as unknown as { columns: number }).columns = 80
const root = await createRoot({
stdout: stdout as unknown as NodeJS.WriteStream,
patchConsole: false,
})
try {
root.render(createElement(AppStateProvider, null, createElement(ImeProbe)))
await waitFor(() => inputState !== undefined)
inputState!.onInput(input, key as Key)
await Bun.sleep(25)
} finally {
root.unmount()
}
return { value: observedValue, cursorOffset: observedCursorOffset }
}
describe('useTextInput IME composition regression (#2018)', () => {
test('composes a backspace-flagged replacement into user-visible text', async () => {
// Telex/VNI compose events arrive flagged as backspace while carrying
// the precomposed replacement character; the visible result is the
// composed word, not a deleted character plus stray text.
const result = await runOnInputScenario({
initialValue: 'xin cha',
input: 'ò',
key: { backspace: true },
})
expect(result.value).toBe('xin chà')
expect(result.cursorOffset).toBe(7)
})
test('composes a delayed standalone combining mark onto its base character', async () => {
// NFD path: the base vowel commits first and the tone mark arrives
// later as its own standalone text event.
const result = await runOnInputScenario({
initialValue: 'tiê',
input: '\u0301',
})
expect(result.value).toBe('tiế')
expect(result.cursorOffset).toBe(3)
})
})
+110 -5
View File
@@ -37,6 +37,64 @@ function mapInput(input_map: Array<[string, InputHandler]>): InputMapper {
}
}
// Any Unicode combining mark (general category M) — covers Vietnamese
// tone/breath marks plus marks outside the U+0300-U+036F block (Hebrew
// points, Devanagari matras, ...) when an IME emits NFD instead of
// precomposed forms.
const COMBINING_MARK_RE = /^\p{M}/u
export type ComposedTextEdit = {
text: string
offset: number
}
/**
* Compose a standalone combining mark (NFD input) onto the character before
* the cursor. Returns null when the input is not a combining mark or there
* is nothing to compose with (#2018).
*/
export function composeCombiningMark(
text: string,
offset: number,
input: string,
): ComposedTextEdit | null {
if (!COMBINING_MARK_RE.test(input) || offset <= 0 || offset > text.length) {
return null
}
const newBefore = (text.slice(0, offset) + input).normalize('NFC')
return {
text: newBefore + text.slice(offset),
offset: newBefore.length,
}
}
/**
* Telex/VNI compose pattern: some IMEs/terminals deliver composition as a
* backspace-flagged event carrying the precomposed replacement character.
* Replace the previous character instead of treating the pair as delete
* plus literal insert. Returns null when the input is not a printable
* non-ASCII replacement (#2018).
*/
export function replacePreviousWithChar(
text: string,
offset: number,
input: string,
): ComposedTextEdit | null {
if (
input.length !== 1 ||
(input.codePointAt(0) ?? 0) <= 127 ||
offset <= 0 ||
offset > text.length
) {
return null
}
const newBefore = text.slice(0, offset - 1) + input
return {
text: (newBefore + text.slice(offset)).normalize('NFC'),
offset: newBefore.normalize('NFC').length,
}
}
export function prepareTextInputEvent(input: string): {
input: string
shouldSubmit: boolean
@@ -84,7 +142,10 @@ export function applyPrintableInput(
return cursor.endOfLine()
}
const text = stripAnsi(input)
// Normalize to NFC so decomposed IME input composes into exactly what
// the cursor's MeasuredText stores, keeping returned offsets aligned
// when normalization shrinks code-unit length (#2018).
const text = stripAnsi(input).normalize('NFC')
if (
!options.modeCharacterIsText &&
cursor.text.length === 0 &&
@@ -112,7 +173,9 @@ export function applyCoalescedDelInput(
let shouldCommit = true
for (let index = 0; index < input.length; index++) {
if (input[index] !== '\x7f') continue
// Both DEL bytes arrive from IMEs/terminals: \x7f is the common
// backspace, \b (Ctrl-H) is emitted by some Vietnamese IME setups.
if (input[index] !== '\x7f' && input[index] !== '\b') continue
if (index > segmentStart) {
const insertedCursor = insert(
@@ -128,7 +191,10 @@ export function applyCoalescedDelInput(
}
let delEnd = index + 1
while (delEnd < input.length && input[delEnd] === '\x7f') {
while (
delEnd < input.length &&
(input[delEnd] === '\x7f' || input[delEnd] === '\b')
) {
delEnd++
}
const delCount = delEnd - index
@@ -631,14 +697,53 @@ export function useTextInput({
return
}
// Vietnamese/CJK IME composition paths (#2018). Both bypass the \r /
// coalesced-Enter handling in prepareTextInputEvent, so they only run
// on inputs that carry no carriage returns.
if (!filteredInput.includes('\r')) {
// Telex/VNI backspace+replacement compose: a backspace-flagged event
// carrying a printable non-ASCII character replaces the previous one
// (a → ă) instead of deleting it.
if (key.backspace && !key.ctrl && !key.meta) {
const replaced = replacePreviousWithChar(
currentCursor.text,
currentCursor.offset,
filteredInput,
)
if (replaced) {
resetKillAccumulation()
resetYankState()
setValue(replaced.text, replaced.offset)
return
}
}
// Standalone combining mark (NFD input): compose onto the preceding
// character rather than inserting a bare mark that later renders,
// measures, and deletes as a separate unit.
const markComposed = composeCombiningMark(
currentCursor.text,
currentCursor.offset,
filteredInput,
)
if (markComposed) {
resetKillAccumulation()
resetYankState()
setValue(markComposed.text, markComposed.offset)
return
}
}
const preparedInput = prepareTextInputEvent(filteredInput)
// Fix Issue #1853: Filter DEL characters that interfere with backspace in SSH/tmux
// In SSH/tmux environments, backspace generates both key events and raw DEL chars
// In SSH/tmux environments, backspace generates both key events and raw DEL chars.
// \b (Ctrl-H) is included because Vietnamese IME setups emit it as the
// backside of their Telex/VNI backspace+replacement compose pattern (#2018).
if (
!key.backspace &&
!key.delete &&
filteredInput.includes('\x7f')
/[\x7f\b]/.test(filteredInput)
) {
let finalChangeContext: TextInputChangeContext | undefined
let modeEntryChangeContext:
+105 -3
View File
@@ -1,9 +1,11 @@
import { EventEmitter } from 'node:events'
import { PassThrough } from 'node:stream'
import { afterEach, describe, expect, mock, test } from 'bun:test'
import { afterEach, describe, expect, jest, mock, test } from 'bun:test'
import type { ParsedKey } from '../parse-keypress.js'
import { createSelectionState } from '../selection.js'
import { PASTE_START } from '../termio/csi.js'
import App from './App.js'
type FakeStdin = NodeJS.ReadStream & {
@@ -33,7 +35,10 @@ function createFakeStdout(): NodeJS.WriteStream {
return stdout
}
function createApp(stdin: NodeJS.ReadStream): App {
function createApp(
stdin: NodeJS.ReadStream,
overrides: { dispatchKeyboardEvent?: (parsedKey: ParsedKey) => void } = {},
): App {
return new App({
children: null,
stdin,
@@ -51,7 +56,7 @@ function createApp(stdin: NodeJS.ReadStream): App {
onOpenHyperlink: () => {},
onMultiClick: () => {},
onSelectionDrag: () => {},
dispatchKeyboardEvent: () => {},
dispatchKeyboardEvent: overrides.dispatchKeyboardEvent ?? (() => {}),
})
}
@@ -117,3 +122,100 @@ describe('App stdin mode setup', () => {
app.handleSetRawMode(false)
})
})
describe('App incomplete-sequence flush timers', () => {
afterEach(() => {
jest.useRealTimers()
})
function createDispatchCollector(): {
dispatched: ParsedKey[]
dispatchKeyboardEvent: (parsedKey: ParsedKey) => void
} {
const dispatched: ParsedKey[] = []
return {
dispatched,
dispatchKeyboardEvent: key => {
dispatched.push(key)
},
}
}
test('holds a lone Escape, then flushes it exactly at NORMAL_TIMEOUT', () => {
jest.useFakeTimers()
const { dispatched, dispatchKeyboardEvent } = createDispatchCollector()
const app = createApp(createFakeStdin(), { dispatchKeyboardEvent })
// The bare ESC is buffered, not emitted as an instant Escape keypress.
app.processInput('\x1b')
expect(dispatched).toEqual([])
jest.advanceTimersByTime(app.NORMAL_TIMEOUT - 1)
expect(dispatched).toEqual([])
jest.advanceTimersByTime(1)
expect(dispatched).toHaveLength(1)
expect(dispatched[0]?.name).toBe('escape')
})
test('holds an Alt-prefixed half and composes it when the rest arrives before the flush', () => {
jest.useFakeTimers()
const { dispatched, dispatchKeyboardEvent } = createDispatchCollector()
const app = createApp(createFakeStdin(), { dispatchKeyboardEvent })
app.processInput('\x1b')
expect(dispatched).toEqual([])
// Continuation lands inside the hold window: ESC + b composes Alt+b.
app.processInput('b')
expect(dispatched).toHaveLength(1)
expect(dispatched[0]?.sequence).toBe('\x1bb')
expect(dispatched[0]?.meta).toBe(true)
// The satisfied hold must not leak a phantom Escape when its timer fires.
jest.advanceTimersByTime(app.NORMAL_TIMEOUT)
expect(dispatched).toHaveLength(1)
})
test('composes a delayed CSI-u chunk that arrives before the flush', () => {
jest.useFakeTimers()
const { dispatched, dispatchKeyboardEvent } = createDispatchCollector()
const app = createApp(createFakeStdin(), { dispatchKeyboardEvent })
app.processInput('\x1b[98')
expect(dispatched).toEqual([])
// IME/CSI-u second half arrives within the hold window: parses as
// kitty Alt+b instead of garbage after a premature flush (#2018).
app.processInput(';3u')
expect(dispatched).toHaveLength(1)
expect(dispatched[0]?.name).toBe('b')
expect(dispatched[0]?.meta).toBe(true)
jest.advanceTimersByTime(app.NORMAL_TIMEOUT)
expect(dispatched).toHaveLength(1)
})
test('holds an incomplete bracketed paste past PASTE_TIMEOUT, then flushes it as one paste', () => {
jest.useFakeTimers()
const { dispatched, dispatchKeyboardEvent } = createDispatchCollector()
const app = createApp(createFakeStdin(), { dispatchKeyboardEvent })
// Paste start + content + truncated CSI tail: tokenizer stays buffered
// and App stays in paste mode, so nothing is emitted yet.
app.processInput(`${PASTE_START}hello\x1b[2`)
expect(dispatched).toEqual([])
jest.advanceTimersByTime(app.PASTE_TIMEOUT - 1)
expect(dispatched).toEqual([])
jest.advanceTimersByTime(1)
expect(dispatched).toHaveLength(1)
expect(dispatched[0]?.isPasted).toBe(true)
expect(dispatched[0]?.sequence).toBe('hello\x1b[2')
// The flush consumed both the paste buffer and the incomplete tail.
expect(app.keyParseState.incomplete).toBe('')
expect(app.keyParseState.mode).toBe('NORMAL')
})
})
+7 -2
View File
@@ -121,8 +121,13 @@ export default class App extends PureComponent<Props, State> {
// where startup input appears frozen when data mode is the default.
stdinMode: 'readable' | 'data' = process.env.OPENCLAUDE_USE_DATA_STDIN === '1' || process.env.OPENCLAUDE_USE_READABLE_STDIN === '0' ? 'data' : 'readable';
// Timeout durations for incomplete sequences (ms)
readonly NORMAL_TIMEOUT = 50; // Short timeout for regular esc sequences
readonly PASTE_TIMEOUT = 500; // Longer timeout for paste operations
// NORMAL_TIMEOUT must exceed IME composition gaps: Vietnamese Telex/VNI
// and CJK IMEs emit multi-byte UTF-8 / CSI-u sequences whose halves can
// arrive with inter-chunk pauses. Flushing at 50ms split those sequences
// into garbage keys (#2018); 300ms gives composition room while keeping
// lone-escape detection acceptable.
readonly NORMAL_TIMEOUT = 300;
readonly PASTE_TIMEOUT = 1000; // Longer timeout for paste operations
// Terminal query/response dispatch. Responses arrive on stdin (parsed
// out by parse-keypress) and are routed to pending promise resolvers.
+17
View File
@@ -186,6 +186,23 @@ function parseKey(keypress: ParsedKey): [Key, string] {
key.shift = true
}
// Printable non-ASCII characters (Vietnamese precomposed letters, CJK,
// accented Latin) arrive as plain text sequences with no keyName-map
// entry, leaving keypress.name empty. Record the character as the name
// so downstream consumers (keyboard-event dispatch, mode handlers) see
// the character instead of an empty string. This runs after the
// nonAlphanumericKeys clear above, and single-code-point names are never
// members of that list, so this cannot cause input to be cleared (#2018).
// Count code points, not UTF-16 units, so one astral code point (e.g.
// emoji) still qualifies.
if (
!keypress.name &&
[...input].length === 1 &&
(input.codePointAt(0) ?? 0) > 127
) {
keypress.name = input
}
return [key, input]
}
+75
View File
@@ -74,3 +74,78 @@ test('preserves Vietnamese UTF-8 input split across stdin chunks', () => {
expect(events.map(event => event.input).join('')).toBe('tiếng Việt')
expect(events.some(event => event.input.includes('\uFFFD'))).toBe(false)
})
test('names precomposed Vietnamese characters typed as plain text', () => {
const event = parseInputEvent('ă')
expect(event.input).toBe('ă')
expect(event.keypress.name).toBe('ă')
})
test('composes NFD Vietnamese text into a single precomposed key', () => {
const event = parseInputEvent('a\u0306')
expect(event.input).toBe('ă')
expect(event.input.codePointAt(0)).toBe(0x0103)
expect(event.keypress.name).toBe('ă')
})
test('names standalone combining marks so they reach input handlers', () => {
const event = parseInputEvent('\u0306')
expect(event.input).toBe('\u0306')
expect(event.keypress.name).toBe('\u0306')
})
test('preserves multi-character Vietnamese words as one input event', () => {
const [items] = parseMultipleKeypresses(INITIAL_STATE, 'chào')
expect(items).toHaveLength(1)
const item = items[0]
expect(item?.kind).toBe('key')
expect((item as ParsedKey).sequence).toBe('chào')
})
test('preserves Vietnamese CSI-u input', () => {
const event = parseInputEvent('\x1b[259u')
expect(event.input).toBe('ă')
expect(event.keypress.name).toBe('ă')
})
test('keeps DEL plus replacement intact for downstream coalescing', () => {
const event = parseInputEvent('\x7fă')
expect(event.input).toBe('\x7fă')
expect(event.key.backspace).toBe(false)
})
test('names astral-plane characters typed as plain text', () => {
// 😀 is one code point but two UTF-16 units; the printable non-ASCII
// branch must count code points so the key still gets a name.
const event = parseInputEvent('😀')
expect(event.input).toBe('😀')
expect(event.keypress.name).toBe('😀')
})
test('InputEvent names astral sequences that reach it without a name', () => {
const unnamedAstral: ParsedKey = {
kind: 'key',
fn: false,
name: '',
ctrl: false,
meta: false,
shift: false,
option: false,
super: false,
sequence: '😀',
raw: '😀',
isPasted: false,
}
const event = new InputEvent(unnamedAstral)
expect(event.input).toBe('😀')
expect(event.keypress.name).toBe('😀')
})
+14 -1
View File
@@ -347,7 +347,11 @@ export function parseMultipleKeypresses(
const mouse = parseMouseEvent(resynthesized)
keys.push(mouse ?? parseKeypress(resynthesized))
} else {
keys.push(parseKeypress(token.value))
// IMEs and some terminals emit decomposed Unicode (NFD) — e.g.
// U+0061 + U+0306 instead of precomposed U+0103 (ă). Normalize
// typed text to NFC here so a decomposed pair parses as one
// composed character instead of arriving as separate keys (#2018).
keys.push(parseKeypress(token.value.normalize('NFC')))
}
}
}
@@ -830,6 +834,15 @@ function parseKeypress(s: string = ''): ParsedKey {
} else if (s.length === 1 && s >= 'A' && s <= 'Z') {
key.name = s.toLowerCase()
key.shift = true
} else if ([...s].length === 1 && (s.codePointAt(0) ?? 0) > 127) {
// Printable non-ASCII character (accented Latin, CJK, astral emoji,
// etc.). Count code points, not UTF-16 units, so a single astral code
// point (UTF-16 length 2, e.g. 😀) is still recognized. Assign it as the
// key name so downstream consumers (keybindings, vim mode, DOM keyboard
// dispatch) can identify it; text insertion still flows through
// `sequence`. Without this, Vietnamese precomposed characters like
// ă/ơ/ư fall through unnamed (#2018).
key.name = s
} else if ((parts = META_KEY_CODE_RE.exec(s))) {
key.meta = true
key.shift = /^[A-Z]$/.test(parts[1]!)