From d62eb70da179352e3f1889ffaed7cdd80b99cd59 Mon Sep 17 00:00:00 2001 From: JATMN Date: Sat, 24 Oct 2026 02:52:27 -0700 Subject: [PATCH] chore(deps): update marked to 18.1.1 (#2382) --- CONTRIBUTING.md | 23 +++++ bun.lock | 4 +- package.json | 2 +- src/commands/copy/copy.tsx | 4 +- src/components/Markdown.tsx | 4 +- .../fixtures/markdownRendering.baseline.json | 88 +++++++++++++++++++ .../fixtures/markdownRendering.fixture.tsx | 68 ++++++++++++++ src/utils/claudemd.ts | 10 ++- src/utils/markdown.test.ts | 45 ++++++++++ src/utils/markdown.ts | 38 ++++++-- 10 files changed, 272 insertions(+), 14 deletions(-) create mode 100644 src/test/fixtures/markdownRendering.baseline.json create mode 100644 src/test/fixtures/markdownRendering.fixture.tsx create mode 100644 src/utils/markdown.test.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a11ece11..9a33cb91d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -180,6 +180,29 @@ Before submitting, run multiple rounds of review on generated code: Self-review up front saves everyone time and reduces back-and-forth during maintainer review. +### Markdown parsing and rendering + +Use the exact `marked` version in `package.json` and its bundled TypeScript types; +import `marked`, `Lexer`, `Token`, or `Tokens` from the package's ESM entry point. +Do not add `@types/marked` or a separate rendering extension. Use +`configureMarked()` before terminal lexing and `formatToken()` / `applyMarkdown()` +for terminal output; strikethrough parsing is deliberately disabled. + +List items can contain `paragraph` children and `checkbox` tokens. Preserve list +markers for paragraph children and omit checkbox glyphs in terminal output. Pass +the previous sibling to `formatToken()` when rendering block sequences so separate +`space` tokens do not duplicate spacing after headings, tables, or indented code. +Indented code has trailing newlines removed for terminal display and `/copy`; +fenced code retains its content. Keep streaming boundary calculations on the +complete lexer token stream, including whitespace and each token's `raw` length. + +CLAUDE.md parsing uses its own `new Lexer({ gfm: false })` to preserve include +paths. When removing a comment-only HTML block, consume its following whitespace +token too; preserve whitespace with visible comment residue and leave code blocks +untouched. When updating `marked`, compare representative terminal, table, list, +code, link, streaming, and memory-file output and run the markdown regression tests. +Regenerate the exact dependency pin and lockfile with `bun add --exact marked@`. + ## AI Agent Guidelines If you are an AI agent (Copilot, Cursor, Claude, etc.) working on this codebase, refer to [AGENTS.md](AGENTS.md) for project-specific coding guidelines, conventions, and validation commands. Following these guidelines will help your contributions align with the project's patterns and reduce review friction. diff --git a/bun.lock b/bun.lock index 889b97660..1cffe990c 100644 --- a/bun.lock +++ b/bun.lock @@ -62,7 +62,7 @@ "knip": "6.42.1", "lodash-es": "4.18.1", "lru-cache": "11.5.3", - "marked": "15.0.12", + "marked": "18.1.1", "p-map": "7.1.1", "picomatch": "4.0.7", "proper-lockfile": "4.1.2", @@ -989,7 +989,7 @@ "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], - "marked": ["marked@15.0.12", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA=="], + "marked": ["marked@18.1.1", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-zEGgrtUG8crk+TrMacJSVABU4DYDDkS8uqgU3ohrn/2fW9C2AEfUrO3m6IhpIpQpl7YcVpiXXC9cSMWDANT3dg=="], "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], diff --git a/package.json b/package.json index baa8c29fa..040b6200a 100644 --- a/package.json +++ b/package.json @@ -142,7 +142,7 @@ "knip": "6.42.1", "lodash-es": "4.18.1", "lru-cache": "11.5.3", - "marked": "15.0.12", + "marked": "18.1.1", "p-map": "7.1.1", "picomatch": "4.0.7", "proper-lockfile": "4.1.2", diff --git a/src/commands/copy/copy.tsx b/src/commands/copy/copy.tsx index d540cdff2..88d99e5f7 100644 --- a/src/commands/copy/copy.tsx +++ b/src/commands/copy/copy.tsx @@ -34,7 +34,9 @@ function extractCodeBlocks(markdown: string): CodeBlock[] { if (token.type === 'code') { const codeToken = token as Tokens.Code; blocks.push({ - code: codeToken.text, + code: codeToken.codeBlockStyle === 'indented' + ? codeToken.text.replace(/\n+$/, '') + : codeToken.text, lang: codeToken.lang }); } diff --git a/src/components/Markdown.tsx b/src/components/Markdown.tsx index 0dadd055f..defe1a063 100644 --- a/src/components/Markdown.tsx +++ b/src/components/Markdown.tsx @@ -140,12 +140,12 @@ function MarkdownBody(t0) { nonTableContent = ""; } }; - for (const token of tokens) { + for (const [index, token] of tokens.entries()) { if (token.type === "table") { flushNonTableContent(); elements.push(); } else { - nonTableContent = nonTableContent + formatToken(token, theme, 0, null, null, highlight); + nonTableContent = nonTableContent + formatToken(token, theme, 0, null, null, highlight, tokens[index - 1]); nonTableContent; } } diff --git a/src/test/fixtures/markdownRendering.baseline.json b/src/test/fixtures/markdownRendering.baseline.json new file mode 100644 index 000000000..fa0b01605 --- /dev/null +++ b/src/test/fixtures/markdownRendering.baseline.json @@ -0,0 +1,88 @@ +{ + "inline": { + "input": "Plain **bold**, *italic*, `code`, ~~approximate~~ and \\*literal\\*.", + "terminal": "Plain bold, italic, code, ~~approximate~~ and *literal*.", + "codes": [] + }, + "headings": { + "input": "# Heading\n\nBody\n\n## Subheading\n\nNext", + "terminal": "Heading\n\nBody\n\nSubheading\n\nNext", + "codes": [] + }, + "blocks": { + "input": "> A quote\n> with **bold**\n\n---\n\nAfter", + "terminal": "▎ A quote\n▎ with bold\n\n---\nAfter", + "codes": [] + }, + "code": { + "input": "Before\n\n```ts\nconst n = 1\nconsole.log(n)\n```\n\nAfter\n\n indented()\n", + "terminal": "Before\n\nconst n = 1\nconsole.log(n)\n\nAfter\n\nindented()", + "codes": [ + { + "code": "const n = 1\nconsole.log(n)", + "lang": "ts" + }, + { + "code": "indented()" + } + ] + }, + "links": { + "input": "[Guide](https://example.com/guide) and and ![alt](image.png)", + "terminal": "https://example.com/guide and dev@example.com and image.png", + "codes": [] + }, + "tight": { + "input": "- First **bold**\n- Second\n continued\n 1. Nested one\n 2. Nested two\n- Last", + "terminal": "- First bold\n- Second\ncontinued\n a. Nested one\n b. Nested two\n- Last", + "codes": [] + }, + "loose": { + "input": "3. First **bold**\n\n4. Second\n\n Another paragraph\n\n5. Last", + "terminal": "3. First bold\n4. Second\n\n4. Another paragraph\n5. Last", + "codes": [] + }, + "tasks": { + "input": "- [ ] Pending\n- [x] Done\n\n- [ ] Loose task\n\n More text", + "terminal": "- Pending\n- Done\n- Loose task\n\n- More text", + "codes": [] + }, + "table": { + "input": "| Name | Value |\n| :--- | ---: |\n| **one** | `two` |\n| 三 | 42 |\n\nAfter", + "terminal": "| Name | Value |\n|------|-------|\n| one | two |\n| 三 | 42 |\n\nAfter", + "codes": [] + }, + "spacing": { + "input": "# Title\n\n\nBody\n\n```\ncode\n```\n\n\nAfter\n", + "terminal": "Title\n\nBody\n\ncode\n\nAfter", + "codes": [ + { + "code": "code", + "lang": "" + } + ] + }, + "listBlocks": { + "input": "- Intro\n\n ```\n code\n ```\n\n > quote\n\n- End", + "terminal": "- Intro\n\ncode\n\n▎ quote\n- End", + "codes": [] + }, + "tableComponent": "┌──────┬───────┐\n│ Name │ Value │\n├──────┼───────┤\n│ one │ two │\n├──────┼───────┤\n│ 三 │ 42 │\n└──────┴───────┘", + "Markdown": "Heading\n\nBody\n\nSubheading\n\nNext\n\n- First bold\n- Second\ncontinued\n a. Nested one\n b. Nested two\n- Last\n\n┌──────┬───────┐\n│ Name │ Value │\n├──────┼───────┤\n│ one │ two │\n├──────┼───────┤\n│ 三 │ 42 │\n└──────┴───────┘\n\nAfter", + "StreamingMarkdown": "Heading\n\nBody\n\nSubheading\n\nNext\n\n- First bold\n- Second\ncontinued\n a. Nested one\n b. Nested two\n- Last\n\n┌──────┬───────┐\n│ Name │ Value │\n├──────┼───────┤\n│ one │ two │\n├──────┼───────┤\n│ 三 │ 42 │\n└──────┴───────┘\n\nAfter", + "memory": { + "input": "# Rules\n\n\n\nUse **Bun**.\n\n```md\n\n@./ignored.md\n```\n\n@./included.md\n", + "content": "# Rules\n\nUse **Bun**.\n\n```md\n\n@./ignored.md\n```\n\n@./included.md\n", + "stripped": true + }, + "memoryFiles": [ + { + "name": "CLAUDE.md", + "content": "# Rules\n\nUse **Bun**.\n\n```md\n\n@./ignored.md\n```\n\n@./included.md\n" + }, + { + "name": "included.md", + "content": "Included rule\n" + } + ] +} diff --git a/src/test/fixtures/markdownRendering.fixture.tsx b/src/test/fixtures/markdownRendering.fixture.tsx new file mode 100644 index 000000000..ef82112cd --- /dev/null +++ b/src/test/fixtures/markdownRendering.fixture.tsx @@ -0,0 +1,68 @@ +import { mkdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { stripVTControlCharacters as stripAnsi } from 'node:util' +import { marked, type Tokens } from 'marked' +import React from 'react' +import { call as copy } from '../../commands/copy/copy.js' +import { Markdown, StreamingMarkdown } from '../../components/Markdown.js' +import { MarkdownTable } from '../../components/MarkdownTable.js' +import { AppStateProvider, getDefaultAppState } from '../../state/AppState.js' +import { processMemoryFile, stripHtmlComments } from '../../utils/claudemd.js' +import { enableConfigs } from '../../utils/config.js' +import { applyMarkdown } from '../../utils/markdown.js' +import { renderToString } from '../../utils/staticRender.js' +import baseline from './markdownRendering.baseline.json' + +// Run in a fresh process: exercise real rendering/configuration without shared +// module mocks or settings mutations from the rest of the unit suite. +enableConfigs() +const output: Record = {} +for (const [name, fixture] of Object.entries(baseline)) { + if (typeof fixture !== 'object' || !('terminal' in fixture)) continue + let codes: { code: string; lang?: string }[] = [] + if (fixture.codes.length) { + const picker = await copy( + () => { throw new Error('Expected the code block picker') }, + { + messages: [{ + type: 'assistant', + message: { content: [{ type: 'text', text: fixture.input }] }, + }], + } as unknown as Parameters[1], + '', + ) + if (!React.isValidElement<{ codeBlocks: typeof codes }>(picker)) { + throw new Error('Missing code block picker') + } + codes = picker.props.codeBlocks + } + output[name] = { + input: fixture.input, + terminal: stripAnsi(applyMarkdown(fixture.input, 'dark')), + codes, + } +} +const table = marked.lexer(baseline.table.input).find(t => t.type === 'table') as Tokens.Table +output.tableComponent = await renderToString( + , 80, +) +const defaultState = getDefaultAppState() +const state = { + ...defaultState, + settings: { ...defaultState.settings, syntaxHighlightingDisabled: true }, +} +const combined = [baseline.headings.input, baseline.tight.input, baseline.table.input].join('\n\n') +for (const Component of [Markdown, StreamingMarkdown]) { + output[Component.name] = await renderToString( + {combined}, 80, + ) +} +output.memory = { input: baseline.memory.input, ...stripHtmlComments(baseline.memory.input) } +const dir = process.argv[2]! +await mkdir(dir, { recursive: true }) +await writeFile(join(dir, 'CLAUDE.md'), baseline.memory.input) +await writeFile(join(dir, 'included.md'), 'Included rule\n') +output.memoryFiles = ( + await processMemoryFile(join(dir, 'CLAUDE.md'), 'Project', new Set(), true) +).map(f => ({ name: f.path.split(/[\\/]/).at(-1), content: f.content })) +console.log(JSON.stringify(output)) diff --git a/src/utils/claudemd.ts b/src/utils/claudemd.ts index a332a3cf0..9cb1990fd 100644 --- a/src/utils/claudemd.ts +++ b/src/utils/claudemd.ts @@ -312,14 +312,20 @@ function stripHtmlCommentsFromTokens(tokens: ReturnType): { let result = '' let stripped = false - for (const token of tokens) { + for (let index = 0; index < tokens.length; index++) { + const token = tokens[index]! if (token.type === 'html') { const trimmed = token.raw.trimStart() if (trimmed.startsWith('')) { // Per CommonMark, a type-2 HTML block ends at the *line* containing // `-->`, so text after `-->` on that line is part of this token. // Strip only the comment spans and keep any residual content. - const residue = stripHtmlCommentSpans(token.raw) + // marked emits trailing blank lines as a separate token. Remove them + // with a comment-only block, or retain them with its visible residue. + const next = tokens[index + 1] + const trailingSpace = next?.type === 'space' ? next.raw : '' + if (trailingSpace) index++ + const residue = stripHtmlCommentSpans(token.raw + trailingSpace) stripped = true if (residue.trim().length > 0) { // Residual content exists (e.g. ` Use bun`): keep it. diff --git a/src/utils/markdown.test.ts b/src/utils/markdown.test.ts new file mode 100644 index 000000000..f500b698e --- /dev/null +++ b/src/utils/markdown.test.ts @@ -0,0 +1,45 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import baseline from '../test/fixtures/markdownRendering.baseline.json' + +let directory: string +let actual: Record + +beforeAll(() => { + directory = mkdtempSync(join(tmpdir(), 'openclaude-markdown-')) + const child = Bun.spawnSync([ + process.execPath, 'run', + join(import.meta.dir, '../test/fixtures/markdownRendering.fixture.tsx'), + directory, + ], { + env: { + ...process.env, + CLAUDE_CONFIG_DIR: join(directory, 'config'), + OPENCLAUDE_CONFIG_DIR: join(directory, 'config'), + }, + stdout: 'pipe', + stderr: 'pipe', + timeout: 15_000, + }) + expect(child.exitCode, child.stderr.toString()).toBe(0) + expect(child.stderr.toString()).not.toContain('[Ink:uncaught]') + actual = JSON.parse(child.stdout.toString()) + // The child has a 15-second deadline; allow another five seconds for setup + // and assertions when the test suite is sharing a busy runner. +}, 20_000) + +afterAll(() => { + if (directory) rmSync(directory, { recursive: true, force: true }) +}) + +// Captured through the same production paths using marked 15.0.12 before the +// dependency change. Assert rendered output, not the lexer's internal shape. +describe('markdown output compatibility', () => { + for (const [name, expected] of Object.entries(baseline)) { + test(name, () => { + expect(actual[name]).toEqual(expected) + }) + } +}) diff --git a/src/utils/markdown.ts b/src/utils/markdown.ts index 50e9928dd..a41f479f2 100644 --- a/src/utils/markdown.ts +++ b/src/utils/markdown.ts @@ -41,7 +41,9 @@ export function applyMarkdown( configureMarked() return marked .lexer(stripPromptXMLTags(content)) - .map(_ => formatToken(_, theme, 0, null, null, highlight)) + .map((token, index, tokens) => + formatToken(token, theme, 0, null, null, highlight, tokens[index - 1]), + ) .join('') .trim() } @@ -53,11 +55,14 @@ export function formatToken( orderedListNumber: number | null = null, parent: Token | null = null, highlight: CliHighlight | null = null, + previousToken?: Token, ): string { switch (token.type) { case 'blockquote': { const inner = (token.tokens ?? []) - .map(_ => formatToken(_, theme, 0, null, null, highlight)) + .map((child, index, tokens) => + formatToken(child, theme, 0, null, null, highlight, tokens[index - 1]), + ) .join('') // Prefix each line with a dim vertical bar. Keep text italic but at // normal brightness — chalk.dim is nearly invisible on dark themes. @@ -70,8 +75,12 @@ export function formatToken( .join(EOL) } case 'code': { + const text = + token.codeBlockStyle === 'indented' + ? token.text.replace(/\n+$/, '') + : token.text if (!highlight) { - return token.text + EOL + return text + EOL } let language = 'plaintext' if (token.lang) { @@ -83,7 +92,7 @@ export function formatToken( ) } } - return highlight.highlight(token.text, { language }) + EOL + return highlight.highlight(text, { language }) + EOL } case 'codespan': { // inline code @@ -176,17 +185,34 @@ export function formatToken( case 'list_item': return (token.tokens ?? []) .map( - _ => - `${' '.repeat(listDepth)}${formatToken(_, theme, listDepth + 1, orderedListNumber, token, highlight)}`, + (child, index, tokens) => + child.type === 'checkbox' + ? '' + : `${' '.repeat(listDepth)}${formatToken(child, theme, listDepth + 1, orderedListNumber, token, highlight, tokens[index - 1])}`, ) .join('') case 'paragraph': + // Loose list children are paragraphs; retain the list marker path. + if (parent?.type === 'list_item') { + return formatToken( + { ...token, type: 'text' }, theme, listDepth, + orderedListNumber, parent, highlight, + ) + } return ( (token.tokens ?? []) .map(_ => formatToken(_, theme, 0, null, null, highlight)) .join('') + EOL ) case 'space': + // These blocks already supply their spacing. Their trailing blank lines + // are separate space tokens in marked 18. + if ( + previousToken && ( + ['heading', 'table', 'html', 'def'].includes(previousToken.type) || + (previousToken.type === 'code' && previousToken.codeBlockStyle === 'indented') + ) + ) return '' return EOL case 'br': return EOL