From c8ef9ea41d8a33ed9926aa4f2da2c53a9e19146a Mon Sep 17 00:00:00 2001 From: Manus AI Date: Wed, 19 Aug 2026 00:09:01 +0000 Subject: [PATCH 1/2] feat: Add opt-in FFM Twemoji support --- packages/markdown-parser-lit/src/render.ts | 13 ++++ packages/markdown-parser-vue/src/render.ts | 12 ++++ packages/markdown-parser/src/core/parser.ts | 23 ++++++- packages/markdown-parser/src/core/render.ts | 9 +++ packages/markdown-parser/src/index.spec.ts | 43 ++++++++++++- packages/markdown-parser/src/index.ts | 1 + packages/markdown-parser/src/rules/twemoji.ts | 60 +++++++++++++++++++ packages/markdown-parser/src/types.ts | 3 + 8 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 packages/markdown-parser/src/rules/twemoji.ts diff --git a/packages/markdown-parser-lit/src/render.ts b/packages/markdown-parser-lit/src/render.ts index f055ad7..48aca18 100644 --- a/packages/markdown-parser-lit/src/render.ts +++ b/packages/markdown-parser-lit/src/render.ts @@ -61,6 +61,19 @@ export function render(nodes?: ASTNode[]): (TemplateResult | string | null)[] { case 'inline_code': return html`${node.content}`; + case 'twemoji': { + const emoji = String(node.emoji ?? ''); + const url = String(node.url ?? '').trim(); + return emoji && isSafeLinkUrl(url) + ? html`${emoji}` + : html`${emoji}`; + } + case 'color_code': { const color = String(node.content ?? ''); if (!isSafeColorValue(color)) return html`${color}`; diff --git a/packages/markdown-parser-vue/src/render.ts b/packages/markdown-parser-vue/src/render.ts index c0c8180..0bbc2b4 100644 --- a/packages/markdown-parser-vue/src/render.ts +++ b/packages/markdown-parser-vue/src/render.ts @@ -35,6 +35,18 @@ export function renderToVue(nodes: ASTNode[]): (VNode | string)[] { return h('del', renderToVue(node.children || [])); case 'inline_code': return h('code', node.content || ''); + case 'twemoji': { + const emoji = String(node.emoji ?? ''); + const url = String(node.url ?? '').trim(); + return emoji && isSafeLinkUrl(url) + ? h('img', { + class: 'emoji', + draggable: false, + alt: emoji, + src: url, + }) + : emoji; + } case 'color_code': { const color = String(node.content ?? ''); if (!isSafeColorValue(color)) return color; diff --git a/packages/markdown-parser/src/core/parser.ts b/packages/markdown-parser/src/core/parser.ts index b4b0b61..cb9d750 100644 --- a/packages/markdown-parser/src/core/parser.ts +++ b/packages/markdown-parser/src/core/parser.ts @@ -16,6 +16,7 @@ const LINKIFY_REGEX = export class MarkdownParser { #blockRuleMap = new Map(); #inlineRuleMap = new Map(); + #fallbackInlineRules: InlineRule[] = []; #idSequence = 0; #isPreflight = false; readonly #maxNestingDepth: number; @@ -57,6 +58,8 @@ export class MarkdownParser { // register inline rule addInlineRule(rule: InlineRule): this { + if (rule.fallback) this.#fallbackInlineRules.push(rule); + // register the rule under its markers for quick lookup during parsing for (const marker of rule.markers) { const rulesForMarker = this.#inlineRuleMap.get(marker) || []; @@ -279,12 +282,13 @@ export class MarkdownParser { // find candidate rules based on the current character const rules = char ? this.#inlineRuleMap.get(char) : undefined; + const fallbackRules = this.#fallbackInlineRules; - if (rules) { + if (rules || fallbackRules.length > 0) { let matched = false; // only check rules that are registered for this marker character - for (const rule of rules) { + for (const rule of rules ?? []) { const result = rule.parse(state, context); if (result) { flushText(state.pos); @@ -296,6 +300,21 @@ export class MarkdownParser { } } + if (!matched) { + // Fallback rules are opt-in and run only after marker-specific rules fail. + for (const rule of fallbackRules ?? []) { + const result = rule.parse(state, context); + if (result) { + flushText(state.pos); + nodes.push(result.node); + state.advance(result.consumedChars); + textStart = state.pos; + matched = true; + break; + } + } + } + // skip the normal character advancement if any rule matched if (matched) continue; } diff --git a/packages/markdown-parser/src/core/render.ts b/packages/markdown-parser/src/core/render.ts index b0f82f9..6bd3cf4 100644 --- a/packages/markdown-parser/src/core/render.ts +++ b/packages/markdown-parser/src/core/render.ts @@ -61,6 +61,15 @@ export function render(nodes?: ASTNode[]): string { case 'inline_code': html += `${node.content ? escapeHtml(node.content) : ''}`; break; + case 'twemoji': { + const emoji = String(node.emoji ?? ''); + const url = String(node.url ?? '').trim(); + html += + emoji && isSafeLinkUrl(url) + ? `${escapeHtml(emoji)}` + : escapeHtml(emoji); + break; + } case 'color_code': { const color = String(node.content ?? ''); if (!isSafeColorValue(color)) { diff --git a/packages/markdown-parser/src/index.spec.ts b/packages/markdown-parser/src/index.spec.ts index 1f828f2..d338189 100644 --- a/packages/markdown-parser/src/index.spec.ts +++ b/packages/markdown-parser/src/index.spec.ts @@ -1,9 +1,11 @@ // @fuyeor/markdown-parser/src/index.spec.ts import { describe, it, expect } from 'vitest'; import { MarkdownParser } from './core/parser'; +import { render } from './core/render'; import { createFuyeorMarkdownParser } from './default'; import { headingRule, codeBlockRule, tableRule } from './rules/blocks'; -import { boldRule, linkRule } from './rules/inlines'; +import { boldRule, inlineCodeRule, linkRule } from './rules/inlines'; +import { twemojiPlugin } from './rules/twemoji'; // build parser const parse = new MarkdownParser() @@ -87,4 +89,43 @@ describe('test @fuyeor/markdown-parser', () => { RangeError, ); }); + + it('supports Twemoji through an explicit opt-in plugin', () => { + const parseWithTwemoji = new MarkdownParser() + .addBlockRule(codeBlockRule) + .addInlineRule(inlineCodeRule) + .use(twemojiPlugin) + .build(); + const ast = parseWithTwemoji('Hello 😀 🇪🇸 👨‍💻'); + const emojiNodes = ast[0].children?.filter( + (node) => node.type === 'twemoji', + ); + + expect(emojiNodes).toHaveLength(3); + expect(emojiNodes?.map((node) => node.emoji)).toEqual(['😀', '🇪🇸', '👨‍💻']); + expect(emojiNodes?.map((node) => node.url)).toEqual([ + 'https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f600.svg', + 'https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f1ea-1f1f8.svg', + 'https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f468-200d-1f4bb.svg', + ]); + expect(render(ast)).toBe( + '

Hello 😀 🇪🇸 👨‍💻

\n', + ); + }); + + it('keeps Twemoji disabled in the default parser', () => { + const ast = createFuyeorMarkdownParser()('😀'); + + expect(ast[0].children?.[0].type).toBe('text'); + }); + + it('does not replace emoji inside inline code', () => { + const parseWithTwemoji = new MarkdownParser() + .addBlockRule(codeBlockRule) + .addInlineRule(inlineCodeRule) + .use(twemojiPlugin) + .build(); + + expect(render(parseWithTwemoji('`😀`'))).toBe('

😀

\n'); + }); }); diff --git a/packages/markdown-parser/src/index.ts b/packages/markdown-parser/src/index.ts index 2451ef8..1cb6cae 100644 --- a/packages/markdown-parser/src/index.ts +++ b/packages/markdown-parser/src/index.ts @@ -19,6 +19,7 @@ export { underlineRule, strikeRule, } from './rules/inlines'; +export { twemojiPlugin, twemojiRule } from './rules/twemoji'; export { createMarkdownParser, createFuyeorMarkdownParser } from './default'; export type { diff --git a/packages/markdown-parser/src/rules/twemoji.ts b/packages/markdown-parser/src/rules/twemoji.ts new file mode 100644 index 0000000..e95a718 --- /dev/null +++ b/packages/markdown-parser/src/rules/twemoji.ts @@ -0,0 +1,60 @@ +// @fuyeor/markdown-parser/src/rules/twemoji.ts +import type { InlineRule, MarkdownPlugin } from '#/types'; + +const TWEMOJI_CDN = 'https://deliver.fuyeor.net/@libs/twemoji-new/svg/'; +const r = String.raw; + +// Match base emoji, flags, keycaps, tag sequences, variation selectors, and ZWJ sequences. +const baseEmoji = r`\p{Emoji}(?:\p{EMod}|[\u{E0020}-\u{E007E}]+\u{E007F}|\uFE0F?\u20E3?)`; +const emojiRegex = new RegExp( + r`\p{RI}{2}|(?![#*\d](?!\uFE0F?\u20E3))${baseEmoji}(?:\u200D${baseEmoji})*`, + 'gu', +); +const emojiStartRegex = /^\p{Emoji}$/u; +const keycapStartRegex = /^(?:[#*\d]\uFE0F?\u20E3)/u; + +function getTwemojiUrl(emoji: string): string { + return `${TWEMOJI_CDN}${Array.from(emoji) + .map((character) => character.codePointAt(0)!.toString(16)) + .filter((codePoint) => codePoint !== 'fe0f') + .join('-')}.svg`; +} + +function isEmojiCandidate(content: string, position: number): boolean { + const codePoint = content.codePointAt(position); + if (codePoint === undefined) return false; + + const character = String.fromCodePoint(codePoint); + if (!emojiStartRegex.test(character)) return false; + + return codePoint > 0x7f || keycapStartRegex.test(content.slice(position)); +} + +// Match one emoji at the current parser position without scanning ordinary text repeatedly. +export const twemojiRule: InlineRule = { + name: 'twemoji', + markers: [], + fallback: true, + parse(state) { + if (!isEmojiCandidate(state.content, state.pos)) return null; + + emojiRegex.lastIndex = state.pos; + const match = emojiRegex.exec(state.content); + if (!match || match.index !== state.pos) return null; + + const emoji = match[0]; + return { + node: { + type: 'twemoji', + emoji, + url: getTwemojiUrl(emoji), + }, + consumedChars: emoji.length, + }; + }, +}; + +// Register Twemoji as an opt-in FFM inline extension. +export const twemojiPlugin: MarkdownPlugin = (parser) => { + parser.addInlineRule(twemojiRule); +}; diff --git a/packages/markdown-parser/src/types.ts b/packages/markdown-parser/src/types.ts index 26ca104..8d1b4f7 100644 --- a/packages/markdown-parser/src/types.ts +++ b/packages/markdown-parser/src/types.ts @@ -21,6 +21,7 @@ export type NodeType = | 'link' | 'hardbreak' | 'hr' + | 'twemoji' | string; export interface ASTNode { @@ -30,6 +31,7 @@ export interface ASTNode { level?: number; lang?: string; url?: string; + emoji?: string; ordered?: boolean; start?: number; headers?: ASTNode[]; @@ -65,6 +67,7 @@ export interface BlockRule { export interface InlineRule { name: string; markers: string[]; + fallback?: boolean; // Returns the generated Node and the number of chars consumed if the match is successful; // returns null if the match fails. parse: ( From 91b9880f2219a5a34cc2a7ed1a84cc167d24de34 Mon Sep 17 00:00:00 2001 From: Manus AI Date: Wed, 19 Aug 2026 01:37:05 +0000 Subject: [PATCH 2/2] feat: Add opt-in FFM LaTeX support --- packages/markdown-parser-lit/src/render.ts | 5 ++ packages/markdown-parser-vue/src/render.ts | 4 ++ packages/markdown-parser/src/core/parser.ts | 5 +- packages/markdown-parser/src/core/render.ts | 6 ++ packages/markdown-parser/src/index.spec.ts | 31 ++++++++ packages/markdown-parser/src/index.ts | 1 + packages/markdown-parser/src/rules/latex.ts | 79 +++++++++++++++++++++ packages/markdown-parser/src/types.ts | 2 + 8 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 packages/markdown-parser/src/rules/latex.ts diff --git a/packages/markdown-parser-lit/src/render.ts b/packages/markdown-parser-lit/src/render.ts index 48aca18..5221779 100644 --- a/packages/markdown-parser-lit/src/render.ts +++ b/packages/markdown-parser-lit/src/render.ts @@ -74,6 +74,11 @@ export function render(nodes?: ASTNode[]): (TemplateResult | string | null)[] { : html`${emoji}`; } + case 'math_inline': + return html`${node.content ?? ''}`; + case 'math_block': + return html`
${node.content ?? ''}
`; + case 'color_code': { const color = String(node.content ?? ''); if (!isSafeColorValue(color)) return html`${color}`; diff --git a/packages/markdown-parser-vue/src/render.ts b/packages/markdown-parser-vue/src/render.ts index 0bbc2b4..8ebab71 100644 --- a/packages/markdown-parser-vue/src/render.ts +++ b/packages/markdown-parser-vue/src/render.ts @@ -47,6 +47,10 @@ export function renderToVue(nodes: ASTNode[]): (VNode | string)[] { }) : emoji; } + case 'math_inline': + return h('span', { class: 'math-inline' }, node.content || ''); + case 'math_block': + return h('div', { class: 'math-block' }, node.content || ''); case 'color_code': { const color = String(node.content ?? ''); if (!isSafeColorValue(color)) return color; diff --git a/packages/markdown-parser/src/core/parser.ts b/packages/markdown-parser/src/core/parser.ts index cb9d750..c9d9aa4 100644 --- a/packages/markdown-parser/src/core/parser.ts +++ b/packages/markdown-parser/src/core/parser.ts @@ -159,7 +159,9 @@ export class MarkdownParser { // ` for code block and inline code firstChar === 96 || // ~ equals ` - firstChar === 126; + firstChar === 126 || + // $ for LaTeX block formulas + firstChar === 36; if (mayInterrupt) { const rules = this.#blockRuleMap.get( @@ -177,6 +179,7 @@ export class MarkdownParser { 'list', 'code_block', 'ffm_blocks', + 'latex_block', ].includes(rule.name) ) { this.#isPreflight = true; diff --git a/packages/markdown-parser/src/core/render.ts b/packages/markdown-parser/src/core/render.ts index 6bd3cf4..05ff69c 100644 --- a/packages/markdown-parser/src/core/render.ts +++ b/packages/markdown-parser/src/core/render.ts @@ -70,6 +70,12 @@ export function render(nodes?: ASTNode[]): string { : escapeHtml(emoji); break; } + case 'math_inline': + html += `${escapeHtml(String(node.content ?? ''))}`; + break; + case 'math_block': + html += `
${escapeHtml(String(node.content ?? ''))}
`; + break; case 'color_code': { const color = String(node.content ?? ''); if (!isSafeColorValue(color)) { diff --git a/packages/markdown-parser/src/index.spec.ts b/packages/markdown-parser/src/index.spec.ts index d338189..3786517 100644 --- a/packages/markdown-parser/src/index.spec.ts +++ b/packages/markdown-parser/src/index.spec.ts @@ -6,6 +6,7 @@ import { createFuyeorMarkdownParser } from './default'; import { headingRule, codeBlockRule, tableRule } from './rules/blocks'; import { boldRule, inlineCodeRule, linkRule } from './rules/inlines'; import { twemojiPlugin } from './rules/twemoji'; +import { latexPlugin } from './rules/latex'; // build parser const parse = new MarkdownParser() @@ -128,4 +129,34 @@ describe('test @fuyeor/markdown-parser', () => { expect(render(parseWithTwemoji('`😀`'))).toBe('

😀

\n'); }); + + it('supports opt-in LaTeX inline and block formulas', () => { + const parseWithLatex = new MarkdownParser() + .addBlockRule(codeBlockRule) + .addInlineRule(inlineCodeRule) + .use(latexPlugin) + .build(); + + expect(render(parseWithLatex('Euler $e^{i\\pi}+1=0$'))).toBe( + '

Euler e^{i\\pi}+1=0

\n', + ); + expect(render(parseWithLatex('$$\nx < y\n$$'))).toBe( + '
x < y
', + ); + expect(render(parseWithLatex('Text\n$$\nx < y\n$$'))).toBe( + '

Text

\n
x < y
', + ); + }); + + it('keeps LaTeX disabled in the default parser and inside inline code', () => { + expect(render(createFuyeorMarkdownParser()('$x$'))).toBe('

$x$

\n'); + + const parseWithLatex = new MarkdownParser() + .addBlockRule(codeBlockRule) + .addInlineRule(inlineCodeRule) + .use(latexPlugin) + .build(); + + expect(render(parseWithLatex('`$x$`'))).toBe('

$x$

\n'); + }); }); diff --git a/packages/markdown-parser/src/index.ts b/packages/markdown-parser/src/index.ts index 1cb6cae..5164368 100644 --- a/packages/markdown-parser/src/index.ts +++ b/packages/markdown-parser/src/index.ts @@ -20,6 +20,7 @@ export { strikeRule, } from './rules/inlines'; export { twemojiPlugin, twemojiRule } from './rules/twemoji'; +export { latexPlugin, latexInlineRule, latexBlockRule } from './rules/latex'; export { createMarkdownParser, createFuyeorMarkdownParser } from './default'; export type { diff --git a/packages/markdown-parser/src/rules/latex.ts b/packages/markdown-parser/src/rules/latex.ts new file mode 100644 index 0000000..47d1e67 --- /dev/null +++ b/packages/markdown-parser/src/rules/latex.ts @@ -0,0 +1,79 @@ +// @fuyeor/markdown-parser/src/rules/latex.ts +import { BlockState } from '#/core/state'; +import type { BlockRule, InlineRule, MarkdownPlugin } from '#/types'; + +function getBlockContent( + state: BlockState, +): { content: string; consumedLines: number } | null { + const firstLine = state.currentLine; + if (!firstLine) return null; + + const firstContent = firstLine.trimStart(); + if (!firstContent.startsWith('$$')) return null; + + const firstRemainder = firstContent.slice(2); + const sameLineEnd = firstRemainder.indexOf('$$'); + if (sameLineEnd !== -1) { + return { + content: firstRemainder.slice(0, sameLineEnd).trim(), + consumedLines: 1, + }; + } + + const contentLines = [firstRemainder]; + for (let offset = 1; state.lineIndex + offset < state.lineCount; offset++) { + const line = state.lines[state.lineIndex + offset]; + const end = line.indexOf('$$'); + if (end !== -1) { + contentLines.push(line.slice(0, end)); + return { + content: contentLines.join('\n').trim(), + consumedLines: offset + 1, + }; + } + contentLines.push(line); + } + + return null; +} + +// Parse single-dollar inline formulas without claiming block delimiters. +export const latexInlineRule: InlineRule = { + name: 'latex_inline', + markers: ['$'], + parse(state) { + if (state.currentChar !== '$' || state.content[state.pos + 1] === '$') + return null; + + const end = state.content.indexOf('$', state.pos + 1); + if (end === -1) return null; + + return { + node: { + type: 'math_inline', + content: state.content.slice(state.pos + 1, end).trim(), + }, + consumedChars: end - state.pos + 1, + }; + }, +}; + +// Parse double-dollar block formulas, including multiline content. +export const latexBlockRule: BlockRule = { + name: 'latex_block', + markers: ['$'], + parse(state) { + const block = getBlockContent(state); + if (!block) return null; + + return { + node: { type: 'math_block', content: block.content }, + consumedLines: block.consumedLines, + }; + }, +}; + +// Register LaTeX as an opt-in FFM extension. +export const latexPlugin: MarkdownPlugin = (parser) => { + parser.addBlockRule(latexBlockRule).addInlineRule(latexInlineRule); +}; diff --git a/packages/markdown-parser/src/types.ts b/packages/markdown-parser/src/types.ts index 8d1b4f7..87496ec 100644 --- a/packages/markdown-parser/src/types.ts +++ b/packages/markdown-parser/src/types.ts @@ -22,6 +22,8 @@ export type NodeType = | 'hardbreak' | 'hr' | 'twemoji' + | 'math_inline' + | 'math_block' | string; export interface ASTNode {