();
+ #fallbackInlineRules: InlineRule[] = [];
#idSequence = 0;
#isPreflight = false;
readonly #maxNestingDepth: number;
@@ -57,6 +58,8 @@ export class MarkdownParser {
// register inline rule
addInlineRule(rule: InlineRule): this {
+ if (rule.fallback) this.#fallbackInlineRules.push(rule);
+
// register the rule under its markers for quick lookup during parsing
for (const marker of rule.markers) {
const rulesForMarker = this.#inlineRuleMap.get(marker) || [];
@@ -156,7 +159,9 @@ export class MarkdownParser {
// ` for code block and inline code
firstChar === 96 ||
// ~ equals `
- firstChar === 126;
+ firstChar === 126 ||
+ // $ for LaTeX block formulas
+ firstChar === 36;
if (mayInterrupt) {
const rules = this.#blockRuleMap.get(
@@ -174,6 +179,7 @@ export class MarkdownParser {
'list',
'code_block',
'ffm_blocks',
+ 'latex_block',
].includes(rule.name)
) {
this.#isPreflight = true;
@@ -279,12 +285,13 @@ export class MarkdownParser {
// find candidate rules based on the current character
const rules = char ? this.#inlineRuleMap.get(char) : undefined;
+ const fallbackRules = this.#fallbackInlineRules;
- if (rules) {
+ if (rules || fallbackRules.length > 0) {
let matched = false;
// only check rules that are registered for this marker character
- for (const rule of rules) {
+ for (const rule of rules ?? []) {
const result = rule.parse(state, context);
if (result) {
flushText(state.pos);
@@ -296,6 +303,21 @@ export class MarkdownParser {
}
}
+ if (!matched) {
+ // Fallback rules are opt-in and run only after marker-specific rules fail.
+ for (const rule of fallbackRules ?? []) {
+ const result = rule.parse(state, context);
+ if (result) {
+ flushText(state.pos);
+ nodes.push(result.node);
+ state.advance(result.consumedChars);
+ textStart = state.pos;
+ matched = true;
+ break;
+ }
+ }
+ }
+
// skip the normal character advancement if any rule matched
if (matched) continue;
}
diff --git a/packages/markdown-parser/src/core/render.ts b/packages/markdown-parser/src/core/render.ts
index b0f82f9..05ff69c 100644
--- a/packages/markdown-parser/src/core/render.ts
+++ b/packages/markdown-parser/src/core/render.ts
@@ -61,6 +61,21 @@ export function render(nodes?: ASTNode[]): string {
case 'inline_code':
html += `${node.content ? escapeHtml(node.content) : ''}`;
break;
+ case 'twemoji': {
+ const emoji = String(node.emoji ?? '');
+ const url = String(node.url ?? '').trim();
+ html +=
+ emoji && isSafeLinkUrl(url)
+ ? `})
`
+ : escapeHtml(emoji);
+ break;
+ }
+ case 'math_inline':
+ html += `${escapeHtml(String(node.content ?? ''))}`;
+ break;
+ case 'math_block':
+ html += `${escapeHtml(String(node.content ?? ''))}
`;
+ break;
case 'color_code': {
const color = String(node.content ?? '');
if (!isSafeColorValue(color)) {
diff --git a/packages/markdown-parser/src/index.spec.ts b/packages/markdown-parser/src/index.spec.ts
index 1f828f2..3786517 100644
--- a/packages/markdown-parser/src/index.spec.ts
+++ b/packages/markdown-parser/src/index.spec.ts
@@ -1,9 +1,12 @@
// @fuyeor/markdown-parser/src/index.spec.ts
import { describe, it, expect } from 'vitest';
import { MarkdownParser } from './core/parser';
+import { render } from './core/render';
import { createFuyeorMarkdownParser } from './default';
import { headingRule, codeBlockRule, tableRule } from './rules/blocks';
-import { boldRule, linkRule } from './rules/inlines';
+import { boldRule, inlineCodeRule, linkRule } from './rules/inlines';
+import { twemojiPlugin } from './rules/twemoji';
+import { latexPlugin } from './rules/latex';
// build parser
const parse = new MarkdownParser()
@@ -87,4 +90,73 @@ describe('test @fuyeor/markdown-parser', () => {
RangeError,
);
});
+
+ it('supports Twemoji through an explicit opt-in plugin', () => {
+ const parseWithTwemoji = new MarkdownParser()
+ .addBlockRule(codeBlockRule)
+ .addInlineRule(inlineCodeRule)
+ .use(twemojiPlugin)
+ .build();
+ const ast = parseWithTwemoji('Hello 😀 🇪🇸 👨💻');
+ const emojiNodes = ast[0].children?.filter(
+ (node) => node.type === 'twemoji',
+ );
+
+ expect(emojiNodes).toHaveLength(3);
+ expect(emojiNodes?.map((node) => node.emoji)).toEqual(['😀', '🇪🇸', '👨💻']);
+ expect(emojiNodes?.map((node) => node.url)).toEqual([
+ 'https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f600.svg',
+ 'https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f1ea-1f1f8.svg',
+ 'https://deliver.fuyeor.net/@libs/twemoji-new/svg/1f468-200d-1f4bb.svg',
+ ]);
+ expect(render(ast)).toBe(
+ 'Hello 
\n',
+ );
+ });
+
+ it('keeps Twemoji disabled in the default parser', () => {
+ const ast = createFuyeorMarkdownParser()('😀');
+
+ expect(ast[0].children?.[0].type).toBe('text');
+ });
+
+ it('does not replace emoji inside inline code', () => {
+ const parseWithTwemoji = new MarkdownParser()
+ .addBlockRule(codeBlockRule)
+ .addInlineRule(inlineCodeRule)
+ .use(twemojiPlugin)
+ .build();
+
+ expect(render(parseWithTwemoji('`😀`'))).toBe('😀
\n');
+ });
+
+ it('supports opt-in LaTeX inline and block formulas', () => {
+ const parseWithLatex = new MarkdownParser()
+ .addBlockRule(codeBlockRule)
+ .addInlineRule(inlineCodeRule)
+ .use(latexPlugin)
+ .build();
+
+ expect(render(parseWithLatex('Euler $e^{i\\pi}+1=0$'))).toBe(
+ 'Euler e^{i\\pi}+1=0
\n',
+ );
+ expect(render(parseWithLatex('$$\nx < y\n$$'))).toBe(
+ 'x < y
',
+ );
+ expect(render(parseWithLatex('Text\n$$\nx < y\n$$'))).toBe(
+ 'Text
\nx < y
',
+ );
+ });
+
+ it('keeps LaTeX disabled in the default parser and inside inline code', () => {
+ expect(render(createFuyeorMarkdownParser()('$x$'))).toBe('$x$
\n');
+
+ const parseWithLatex = new MarkdownParser()
+ .addBlockRule(codeBlockRule)
+ .addInlineRule(inlineCodeRule)
+ .use(latexPlugin)
+ .build();
+
+ expect(render(parseWithLatex('`$x$`'))).toBe('$x$
\n');
+ });
});
diff --git a/packages/markdown-parser/src/index.ts b/packages/markdown-parser/src/index.ts
index 2451ef8..5164368 100644
--- a/packages/markdown-parser/src/index.ts
+++ b/packages/markdown-parser/src/index.ts
@@ -19,6 +19,8 @@ export {
underlineRule,
strikeRule,
} from './rules/inlines';
+export { twemojiPlugin, twemojiRule } from './rules/twemoji';
+export { latexPlugin, latexInlineRule, latexBlockRule } from './rules/latex';
export { createMarkdownParser, createFuyeorMarkdownParser } from './default';
export type {
diff --git a/packages/markdown-parser/src/rules/latex.ts b/packages/markdown-parser/src/rules/latex.ts
new file mode 100644
index 0000000..47d1e67
--- /dev/null
+++ b/packages/markdown-parser/src/rules/latex.ts
@@ -0,0 +1,79 @@
+// @fuyeor/markdown-parser/src/rules/latex.ts
+import { BlockState } from '#/core/state';
+import type { BlockRule, InlineRule, MarkdownPlugin } from '#/types';
+
+function getBlockContent(
+ state: BlockState,
+): { content: string; consumedLines: number } | null {
+ const firstLine = state.currentLine;
+ if (!firstLine) return null;
+
+ const firstContent = firstLine.trimStart();
+ if (!firstContent.startsWith('$$')) return null;
+
+ const firstRemainder = firstContent.slice(2);
+ const sameLineEnd = firstRemainder.indexOf('$$');
+ if (sameLineEnd !== -1) {
+ return {
+ content: firstRemainder.slice(0, sameLineEnd).trim(),
+ consumedLines: 1,
+ };
+ }
+
+ const contentLines = [firstRemainder];
+ for (let offset = 1; state.lineIndex + offset < state.lineCount; offset++) {
+ const line = state.lines[state.lineIndex + offset];
+ const end = line.indexOf('$$');
+ if (end !== -1) {
+ contentLines.push(line.slice(0, end));
+ return {
+ content: contentLines.join('\n').trim(),
+ consumedLines: offset + 1,
+ };
+ }
+ contentLines.push(line);
+ }
+
+ return null;
+}
+
+// Parse single-dollar inline formulas without claiming block delimiters.
+export const latexInlineRule: InlineRule = {
+ name: 'latex_inline',
+ markers: ['$'],
+ parse(state) {
+ if (state.currentChar !== '$' || state.content[state.pos + 1] === '$')
+ return null;
+
+ const end = state.content.indexOf('$', state.pos + 1);
+ if (end === -1) return null;
+
+ return {
+ node: {
+ type: 'math_inline',
+ content: state.content.slice(state.pos + 1, end).trim(),
+ },
+ consumedChars: end - state.pos + 1,
+ };
+ },
+};
+
+// Parse double-dollar block formulas, including multiline content.
+export const latexBlockRule: BlockRule = {
+ name: 'latex_block',
+ markers: ['$'],
+ parse(state) {
+ const block = getBlockContent(state);
+ if (!block) return null;
+
+ return {
+ node: { type: 'math_block', content: block.content },
+ consumedLines: block.consumedLines,
+ };
+ },
+};
+
+// Register LaTeX as an opt-in FFM extension.
+export const latexPlugin: MarkdownPlugin = (parser) => {
+ parser.addBlockRule(latexBlockRule).addInlineRule(latexInlineRule);
+};
diff --git a/packages/markdown-parser/src/rules/twemoji.ts b/packages/markdown-parser/src/rules/twemoji.ts
new file mode 100644
index 0000000..e95a718
--- /dev/null
+++ b/packages/markdown-parser/src/rules/twemoji.ts
@@ -0,0 +1,60 @@
+// @fuyeor/markdown-parser/src/rules/twemoji.ts
+import type { InlineRule, MarkdownPlugin } from '#/types';
+
+const TWEMOJI_CDN = 'https://deliver.fuyeor.net/@libs/twemoji-new/svg/';
+const r = String.raw;
+
+// Match base emoji, flags, keycaps, tag sequences, variation selectors, and ZWJ sequences.
+const baseEmoji = r`\p{Emoji}(?:\p{EMod}|[\u{E0020}-\u{E007E}]+\u{E007F}|\uFE0F?\u20E3?)`;
+const emojiRegex = new RegExp(
+ r`\p{RI}{2}|(?)${baseEmoji}(?:\u200D${baseEmoji})*`,
+ 'gu',
+);
+const emojiStartRegex = /^\p{Emoji}$/u;
+const keycapStartRegex = /^(?:[#*\d]\uFE0F?\u20E3)/u;
+
+function getTwemojiUrl(emoji: string): string {
+ return `${TWEMOJI_CDN}${Array.from(emoji)
+ .map((character) => character.codePointAt(0)!.toString(16))
+ .filter((codePoint) => codePoint !== 'fe0f')
+ .join('-')}.svg`;
+}
+
+function isEmojiCandidate(content: string, position: number): boolean {
+ const codePoint = content.codePointAt(position);
+ if (codePoint === undefined) return false;
+
+ const character = String.fromCodePoint(codePoint);
+ if (!emojiStartRegex.test(character)) return false;
+
+ return codePoint > 0x7f || keycapStartRegex.test(content.slice(position));
+}
+
+// Match one emoji at the current parser position without scanning ordinary text repeatedly.
+export const twemojiRule: InlineRule = {
+ name: 'twemoji',
+ markers: [],
+ fallback: true,
+ parse(state) {
+ if (!isEmojiCandidate(state.content, state.pos)) return null;
+
+ emojiRegex.lastIndex = state.pos;
+ const match = emojiRegex.exec(state.content);
+ if (!match || match.index !== state.pos) return null;
+
+ const emoji = match[0];
+ return {
+ node: {
+ type: 'twemoji',
+ emoji,
+ url: getTwemojiUrl(emoji),
+ },
+ consumedChars: emoji.length,
+ };
+ },
+};
+
+// Register Twemoji as an opt-in FFM inline extension.
+export const twemojiPlugin: MarkdownPlugin = (parser) => {
+ parser.addInlineRule(twemojiRule);
+};
diff --git a/packages/markdown-parser/src/types.ts b/packages/markdown-parser/src/types.ts
index 26ca104..87496ec 100644
--- a/packages/markdown-parser/src/types.ts
+++ b/packages/markdown-parser/src/types.ts
@@ -21,6 +21,9 @@ export type NodeType =
| 'link'
| 'hardbreak'
| 'hr'
+ | 'twemoji'
+ | 'math_inline'
+ | 'math_block'
| string;
export interface ASTNode {
@@ -30,6 +33,7 @@ export interface ASTNode {
level?: number;
lang?: string;
url?: string;
+ emoji?: string;
ordered?: boolean;
start?: number;
headers?: ASTNode[];
@@ -65,6 +69,7 @@ export interface BlockRule {
export interface InlineRule {
name: string;
markers: string[];
+ fallback?: boolean;
// Returns the generated Node and the number of chars consumed if the match is successful;
// returns null if the match fails.
parse: (