Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions packages/markdown-parser-lit/src/render.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,24 @@ export function render(nodes?: ASTNode[]): (TemplateResult | string | null)[] {
case 'inline_code':
return html`<code>${node.content}</code>`;

case 'twemoji': {
const emoji = String(node.emoji ?? '');
const url = String(node.url ?? '').trim();
return emoji && isSafeLinkUrl(url)
? html`<img
class="emoji"
draggable="false"
alt=${emoji}
src=${url}
/>`
: html`${emoji}`;
}

case 'math_inline':
return html`<span class="math-inline">${node.content ?? ''}</span>`;
case 'math_block':
return html`<div class="math-block">${node.content ?? ''}</div>`;

case 'color_code': {
const color = String(node.content ?? '');
if (!isSafeColorValue(color)) return html`${color}`;
Expand Down
16 changes: 16 additions & 0 deletions packages/markdown-parser-vue/src/render.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,22 @@ export function renderToVue(nodes: ASTNode[]): (VNode | string)[] {
return h('del', renderToVue(node.children || []));
case 'inline_code':
return h('code', node.content || '');
case 'twemoji': {
const emoji = String(node.emoji ?? '');
const url = String(node.url ?? '').trim();
return emoji && isSafeLinkUrl(url)
? h('img', {
class: 'emoji',
draggable: false,
alt: emoji,
src: url,
})
: emoji;
}
case 'math_inline':
return h('span', { class: 'math-inline' }, node.content || '');
case 'math_block':
return h('div', { class: 'math-block' }, node.content || '');
case 'color_code': {
const color = String(node.content ?? '');
if (!isSafeColorValue(color)) return color;
Expand Down
28 changes: 25 additions & 3 deletions packages/markdown-parser/src/core/parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ const LINKIFY_REGEX =
export class MarkdownParser {
#blockRuleMap = new Map<string, BlockRule[]>();
#inlineRuleMap = new Map<string, InlineRule[]>();
#fallbackInlineRules: InlineRule[] = [];
#idSequence = 0;
#isPreflight = false;
readonly #maxNestingDepth: number;
Expand Down Expand Up @@ -57,6 +58,8 @@ export class MarkdownParser {

// register inline rule
addInlineRule(rule: InlineRule): this {
if (rule.fallback) this.#fallbackInlineRules.push(rule);

// register the rule under its markers for quick lookup during parsing
for (const marker of rule.markers) {
const rulesForMarker = this.#inlineRuleMap.get(marker) || [];
Expand Down Expand Up @@ -156,7 +159,9 @@ export class MarkdownParser {
// ` for code block and inline code
firstChar === 96 ||
// ~ equals `
firstChar === 126;
firstChar === 126 ||
// $ for LaTeX block formulas
firstChar === 36;

if (mayInterrupt) {
const rules = this.#blockRuleMap.get(
Expand All @@ -174,6 +179,7 @@ export class MarkdownParser {
'list',
'code_block',
'ffm_blocks',
'latex_block',
].includes(rule.name)
) {
this.#isPreflight = true;
Expand Down Expand Up @@ -279,12 +285,13 @@ export class MarkdownParser {

// find candidate rules based on the current character
const rules = char ? this.#inlineRuleMap.get(char) : undefined;
const fallbackRules = this.#fallbackInlineRules;

if (rules) {
if (rules || fallbackRules.length > 0) {
let matched = false;

// only check rules that are registered for this marker character
for (const rule of rules) {
for (const rule of rules ?? []) {
const result = rule.parse(state, context);
if (result) {
flushText(state.pos);
Expand All @@ -296,6 +303,21 @@ export class MarkdownParser {
}
}

if (!matched) {
// Fallback rules are opt-in and run only after marker-specific rules fail.
for (const rule of fallbackRules ?? []) {
const result = rule.parse(state, context);
if (result) {
flushText(state.pos);
nodes.push(result.node);
state.advance(result.consumedChars);
textStart = state.pos;
matched = true;
break;
}
}
}

// skip the normal character advancement if any rule matched
if (matched) continue;
}
Expand Down
15 changes: 15 additions & 0 deletions packages/markdown-parser/src/core/render.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,21 @@ export function render(nodes?: ASTNode[]): string {
case 'inline_code':
html += `<code>${node.content ? escapeHtml(node.content) : ''}</code>`;
break;
case 'twemoji': {
const emoji = String(node.emoji ?? '');
const url = String(node.url ?? '').trim();
html +=
emoji && isSafeLinkUrl(url)
? `<img class="emoji" draggable="false" alt="${escapeHtml(emoji)}" src="${escapeHtml(url)}"/>`
: escapeHtml(emoji);
break;
}
case 'math_inline':
html += `<span class="math-inline">${escapeHtml(String(node.content ?? ''))}</span>`;
break;
case 'math_block':
html += `<div class="math-block">${escapeHtml(String(node.content ?? ''))}</div>`;
break;
case 'color_code': {
const color = String(node.content ?? '');
if (!isSafeColorValue(color)) {
Expand Down
74 changes: 73 additions & 1 deletion packages/markdown-parser/src/index.spec.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
// @fuyeor/markdown-parser/src/index.spec.ts
import { describe, it, expect } from 'vitest';
import { MarkdownParser } from './core/parser';
import { render } from './core/render';
import { createFuyeorMarkdownParser } from './default';
import { headingRule, codeBlockRule, tableRule } from './rules/blocks';
import { boldRule, linkRule } from './rules/inlines';
import { boldRule, inlineCodeRule, linkRule } from './rules/inlines';
import { twemojiPlugin } from './rules/twemoji';
import { latexPlugin } from './rules/latex';

// build parser
const parse = new MarkdownParser()
Expand Down Expand Up @@ -87,4 +90,73 @@ describe('test @fuyeor/markdown-parser', () => {
RangeError,
);
});

it('supports Twemoji through an explicit opt-in plugin', () => {
const parseWithTwemoji = new MarkdownParser()
.addBlockRule(codeBlockRule)
.addInlineRule(inlineCodeRule)
.use(twemojiPlugin)
.build();
const ast = parseWithTwemoji('Hello 😀 🇪🇸 👨‍💻');
const emojiNodes = ast[0].children?.filter(
(node) => node.type === 'twemoji',
);

expect(emojiNodes).toHaveLength(3);
expect(emojiNodes?.map((node) => node.emoji)).toEqual(['😀', '🇪🇸', '👨‍💻']);
expect(emojiNodes?.map((node) => node.url)).toEqual([
'https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f600.svg',
'https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f1ea-1f1f8.svg',
'https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f468-200d-1f4bb.svg',
]);
expect(render(ast)).toBe(
'<p>Hello <img class="emoji" draggable="false" alt="😀" src="https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f600.svg"/> <img class="emoji" draggable="false" alt="🇪🇸" src="https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f1ea-1f1f8.svg"/> <img class="emoji" draggable="false" alt="👨‍💻" src="https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f468-200d-1f4bb.svg"/></p>\n',
);
});

it('keeps Twemoji disabled in the default parser', () => {
const ast = createFuyeorMarkdownParser()('😀');

expect(ast[0].children?.[0].type).toBe('text');
});

it('does not replace emoji inside inline code', () => {
const parseWithTwemoji = new MarkdownParser()
.addBlockRule(codeBlockRule)
.addInlineRule(inlineCodeRule)
.use(twemojiPlugin)
.build();

expect(render(parseWithTwemoji('`😀`'))).toBe('<p><code>😀</code></p>\n');
});

it('supports opt-in LaTeX inline and block formulas', () => {
const parseWithLatex = new MarkdownParser()
.addBlockRule(codeBlockRule)
.addInlineRule(inlineCodeRule)
.use(latexPlugin)
.build();

expect(render(parseWithLatex('Euler $e^{i\\pi}+1=0$'))).toBe(
'<p>Euler <span class="math-inline">e^{i\\pi}+1=0</span></p>\n',
);
expect(render(parseWithLatex('$$\nx < y\n$$'))).toBe(
'<div class="math-block">x &lt; y</div>',
);
expect(render(parseWithLatex('Text\n$$\nx < y\n$$'))).toBe(
'<p>Text</p>\n<div class="math-block">x &lt; y</div>',
);
});

it('keeps LaTeX disabled in the default parser and inside inline code', () => {
expect(render(createFuyeorMarkdownParser()('$x$'))).toBe('<p>$x$</p>\n');

const parseWithLatex = new MarkdownParser()
.addBlockRule(codeBlockRule)
.addInlineRule(inlineCodeRule)
.use(latexPlugin)
.build();

expect(render(parseWithLatex('`$x$`'))).toBe('<p><code>$x$</code></p>\n');
});
});
2 changes: 2 additions & 0 deletions packages/markdown-parser/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ export {
underlineRule,
strikeRule,
} from './rules/inlines';
export { twemojiPlugin, twemojiRule } from './rules/twemoji';
export { latexPlugin, latexInlineRule, latexBlockRule } from './rules/latex';
export { createMarkdownParser, createFuyeorMarkdownParser } from './default';

export type {
Expand Down
79 changes: 79 additions & 0 deletions packages/markdown-parser/src/rules/latex.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// @fuyeor/markdown-parser/src/rules/latex.ts
import { BlockState } from '#/core/state';
import type { BlockRule, InlineRule, MarkdownPlugin } from '#/types';

function getBlockContent(
state: BlockState,
): { content: string; consumedLines: number } | null {
const firstLine = state.currentLine;
if (!firstLine) return null;

const firstContent = firstLine.trimStart();
if (!firstContent.startsWith('$$')) return null;

const firstRemainder = firstContent.slice(2);
const sameLineEnd = firstRemainder.indexOf('$$');
if (sameLineEnd !== -1) {
return {
content: firstRemainder.slice(0, sameLineEnd).trim(),
consumedLines: 1,
};
}

const contentLines = [firstRemainder];
for (let offset = 1; state.lineIndex + offset < state.lineCount; offset++) {
const line = state.lines[state.lineIndex + offset];
const end = line.indexOf('$$');
if (end !== -1) {
contentLines.push(line.slice(0, end));
return {
content: contentLines.join('\n').trim(),
consumedLines: offset + 1,
};
}
contentLines.push(line);
}

return null;
}

// Parse single-dollar inline formulas without claiming block delimiters.
export const latexInlineRule: InlineRule = {
name: 'latex_inline',
markers: ['$'],
parse(state) {
if (state.currentChar !== '$' || state.content[state.pos + 1] === '$')
return null;

const end = state.content.indexOf('$', state.pos + 1);
if (end === -1) return null;

return {
node: {
type: 'math_inline',
content: state.content.slice(state.pos + 1, end).trim(),
},
consumedChars: end - state.pos + 1,
};
},
};

// Parse double-dollar block formulas, including multiline content.
export const latexBlockRule: BlockRule = {
name: 'latex_block',
markers: ['$'],
parse(state) {
const block = getBlockContent(state);
if (!block) return null;

return {
node: { type: 'math_block', content: block.content },
consumedLines: block.consumedLines,
};
},
};

// Register LaTeX as an opt-in FFM extension.
export const latexPlugin: MarkdownPlugin = (parser) => {
parser.addBlockRule(latexBlockRule).addInlineRule(latexInlineRule);
};
60 changes: 60 additions & 0 deletions packages/markdown-parser/src/rules/twemoji.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// @fuyeor/markdown-parser/src/rules/twemoji.ts
import type { InlineRule, MarkdownPlugin } from '#/types';

const TWEMOJI_CDN = 'https://deliver.fuyeor.net/@libs/twemoji-new/svg/';
const r = String.raw;

// Match base emoji, flags, keycaps, tag sequences, variation selectors, and ZWJ sequences.
const baseEmoji = r`\p{Emoji}(?:\p{EMod}|[\u{E0020}-\u{E007E}]+\u{E007F}|\uFE0F?\u20E3?)`;
const emojiRegex = new RegExp(
r`\p{RI}{2}|(?![#*\d](?!\uFE0F?\u20E3))${baseEmoji}(?:\u200D${baseEmoji})*`,
'gu',
);
const emojiStartRegex = /^\p{Emoji}$/u;
const keycapStartRegex = /^(?:[#*\d]\uFE0F?\u20E3)/u;

function getTwemojiUrl(emoji: string): string {
return `${TWEMOJI_CDN}${Array.from(emoji)
.map((character) => character.codePointAt(0)!.toString(16))
.filter((codePoint) => codePoint !== 'fe0f')
.join('-')}.svg`;
}

function isEmojiCandidate(content: string, position: number): boolean {
const codePoint = content.codePointAt(position);
if (codePoint === undefined) return false;

const character = String.fromCodePoint(codePoint);
if (!emojiStartRegex.test(character)) return false;

return codePoint > 0x7f || keycapStartRegex.test(content.slice(position));
}

// Match one emoji at the current parser position without scanning ordinary text repeatedly.
export const twemojiRule: InlineRule = {
name: 'twemoji',
markers: [],
fallback: true,
parse(state) {
if (!isEmojiCandidate(state.content, state.pos)) return null;

emojiRegex.lastIndex = state.pos;
const match = emojiRegex.exec(state.content);
if (!match || match.index !== state.pos) return null;

const emoji = match[0];
return {
node: {
type: 'twemoji',
emoji,
url: getTwemojiUrl(emoji),
},
consumedChars: emoji.length,
};
},
};

// Register Twemoji as an opt-in FFM inline extension.
export const twemojiPlugin: MarkdownPlugin = (parser) => {
parser.addInlineRule(twemojiRule);
};
Loading
Loading