From b2a1fe5b68885ca8c0e167933a1671979c06d932 Mon Sep 17 00:00:00 2001 From: Jeremy Howard Date: Tue, 25 Aug 2026 14:23:13 +1000 Subject: [PATCH] Make fastpylight highlighting optional, shrinking the wheel from 8.8MB to 0.95MB --- Cargo.toml | 3 --- DEV.md | 7 ++++++ README.md | 12 +++++++--- pyproject.toml | 1 + python/mdhtml/__init__.py | 2 +- python/mdhtml/export.py | 27 +++++++++++++++++++--- python/mdhtml/md2html.py | 7 +++--- python/mdhtml/viewmd.py | 5 ++-- src/chunk.rs | 5 ++-- src/export_html.rs | 48 ++++++++++++++++++++++++++++----------- src/lib.rs | 6 +++-- src/python.rs | 30 ++++++++---------------- src/wikitext.rs | 27 ++++++++-------------- tests/test_export.py | 12 ++++++++++ tests/wikitext.rs | 6 +++-- 15 files changed, 124 insertions(+), 74 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e14ae09..62fce03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,14 +28,11 @@ codegen-units = 16 [dependencies] base64 = "0.23.0" fast5ever = { git = "https://github.com/AnswerDotAI/fast5ever" } -fastpylight = { git = "https://github.com/AnswerDotAI/fastpylight", default-features = false, features = ["standard-languages", "themes"], optional = true } html-escape = ">=0.2" pyo3 = { version = ">=0.28", optional = true } unicode-properties = "0.1" [features] -default = ["hl"] -hl = ["dep:fastpylight"] python = ["dep:pyo3"] extension-module = ["python", "pyo3/extension-module"] diff --git a/DEV.md b/DEV.md index e67009c..c2eab1a 100644 --- a/DEV.md +++ b/DEV.md @@ -96,6 +96,13 @@ gen_docs() Rust renders provisional markup and does no HTML parsing. `python/mdhtml/__init__.py` sends that markup through `mdhtml2dom`, backed by [fast5ever](https://github.com/AnswerDotAI/fast5ever) (html5ever with an arena DOM and Python bindings), so parsing, tree construction, and serialization are the WHATWG algorithms as one engine spells them. The README describes the public API and `docs/DIALECT.md` defines the resulting DOM contract. +Non-Markdown syntax highlighting is an optional Python-layer adapter rather +than a Rust dependency. Python imports fastpylight lazily and passes its result +through `HtmlExportOptions::hl_fn`; the base Rust crate therefore carries no +fastpylight or tree-sitter code. Without the `hl` extra, `mdhtml2html` leaves +those code blocks plain and reports a warning, while Markdown fences continue +to use mdhtml's own highlighter. + `ops()` is the semantic-operation view over that DOM. Its traversal follows both ordinary children and inert `template.content`, returning live fast5ever nodes so source-specific pipelines can detach or replace operations without adding mutation policy to mdhtml. ## Render callbacks diff --git a/README.md b/README.md index d1db7df..c32db0c 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,13 @@ Install via pip to get both the Python API and the `md2mdhtml` CLI: pip install mdhtml ``` +The base install has no syntax-highlighter dependency. Install `mdhtml[hl]` for +fastpylight highlighting and the theme assets used by `md2html` and `viewmd`: + +```bash +pip install 'mdhtml[hl]' +``` + The CLI reads Markdown from stdin or from an optional file path and writes an MDHTML fragment to stdout: ```bash @@ -72,7 +79,7 @@ md2mdhtml --implicit_figures input.md > out.html md2mdhtml --no-bare_autolinks input.md > out.html ``` -`md2html` goes the rest of the way, lowering that fragment to a finished HTML page: references baked, headings and captions numbered, code highlighted (```` ```markdown ```` fences by mdhtml itself, everything else by fastpylight), mustache tokens shown as styled pills, and the assets those features need (`dialect_css`, light and dark fastpylight themes, KaTeX plus `math_js`) composed into the page. With no `--out` it writes the page under `~/.cache/md2html/` and opens it in a browser, inlining local images so the page renders from anywhere; piped, it writes to stdout instead, and `--out -` forces that even at a terminal. `--fragment` emits the body alone. `--frontmatter` recognizes a leading metadata block (see below), and ```mermaid fences become diagrams drawn in place by mermaid.js. References default to `--refs=ids`, which shows each reference's target id and never fails on a draft; `--refs=resolve` numbers them and raises on a broken one, and `--refs=lenient` numbers what it can and warns about the rest. +`md2html` goes the rest of the way, lowering that fragment to a finished HTML page: references baked, headings and captions numbered, code highlighted (```` ```markdown ```` fences by mdhtml itself, everything else by the optional fastpylight extra), mustache tokens shown as styled pills, and the assets those features need (`dialect_css`, light and dark fastpylight themes, KaTeX plus `math_js`) composed into the page. With no `--out` it writes the page under `~/.cache/md2html/` and opens it in a browser, inlining local images so the page renders from anywhere; piped, it writes to stdout instead, and `--out -` forces that even at a terminal. `--fragment` emits the body alone. `--frontmatter` recognizes a leading metadata block (see below), and ```mermaid fences become diagrams drawn in place by mermaid.js. References default to `--refs=ids`, which shows each reference's target id and never fails on a draft; `--refs=resolve` numbers them and raises on a broken one, and `--refs=lenient` numbers what it can and warns about the rest. ```bash md2html input.md @@ -365,7 +372,7 @@ The result is still a body fragment (a str subclass carrying a `warnings` list; - `{=html}` raw data is decoded and spliced in place; raw data for other formats is removed. Malformed payloads are dropped with a warning. - A `colwidths` attribute lowers to a ``; `fr` values share the width remaining after fixed lengths. - A `width` attribute on a table lowers to an inline style width (bare number = px; invalid values stay visible); it merges last, so it beats `colwidths`' `width:100%`. -- Code blocks with a language are highlighted (natively, via the statically linked fastpylight engine): `hl='spans'` (default) emits `hl-*` classed spans, `hl='api'` wraps the block in the `` element for the CSS Custom Highlight API, and `hl=None` leaves code untouched. Two per-block hooks customize this: `hl_lang(text, lang)` may return a corrected language before highlighting (e.g. mapping a `%%sql` first line to `sql`), and `code_wrap(html, lang, text)` may return replacement markup for the finished block (a copy-button wrapper, a mermaid `pre`). +- Code blocks with a language are highlighted through the optional [fastpylight](https://github.com/AnswerDotAI/fastpylight) package (`pip install 'mdhtml[hl]'`): `hl='spans'` (default) emits `hl-*` classed spans, `hl='api'` wraps the block in the `` element for the CSS Custom Highlight API, and `hl=None` leaves code untouched. Without fastpylight installed, code blocks render plain and a warning reports it (```` ```markdown ```` fences always self-highlight, with no dependency). Rust consumers get the same seam as the `hl_fn` slot on `HtmlExportOptions`: a `(code, lang, mode)` hook returning highlighted markup. Two per-block hooks customize this: `hl_lang(text, lang)` may return a corrected language before highlighting (e.g. mapping a `%%sql` first line to `sql`), and `code_wrap(html, lang, text)` may return replacement markup for the finished block (a copy-button wrapper, a mermaid `pre`). - `toc=True` prepends a `