From 92ad2c4de6d0bcd1082a0dd6d610bda8fce4f299 Mon Sep 17 00:00:00 2001 From: Jeremy Howard Date: Thu, 27 Aug 2026 10:34:43 +1000 Subject: [PATCH] Add gh_ids: derive heading ids by GitHub's rules --- docs/DIALECT.md | 4 +++- python/mdhtml/export.py | 6 ++++-- python/mdhtml/md2html.py | 4 +++- src/export_html.rs | 34 +++++++++++++++++++++++++++------- src/python.rs | 4 +++- tests/test_export.py | 29 +++++++++++++++++++++++++++++ 6 files changed, 69 insertions(+), 12 deletions(-) diff --git a/docs/DIALECT.md b/docs/DIALECT.md index caeaa40..5ddb9ba 100644 --- a/docs/DIALECT.md +++ b/docs/DIALECT.md @@ -95,7 +95,9 @@ Headings use `h1` through `h6`; paragraphs, thematic breaks, and block quotes us ## Identifiers and rendering options -Automatic heading ids are an export concern, not part of the parse: `md2mdhtml` emits only authored ids, so identical fragments render identically wherever they later appear. `mdhtml2html`'s `auto_ids` option (on by default there) derives an id for each heading without one, and any converter that derives section ids must use the same rules: text is lowercased; whitespace becomes `-`; characters other than letters, numbers, `_`, `-`, and `.` are removed; leading nonletters are removed; and an empty result becomes `section`. Duplicate ids receive `-1`, `-2`, and so on, deduplicated within one export. Explicit ids win and participate in duplicate detection. So `## Hello, world!` exports as `

Hello, world!

`. Embedders rendering several fragments into one page pass `auto_ids=False`, since per-fragment derivation cannot see a sibling fragment's ids. +Automatic heading ids are an export concern, not part of the parse: `md2mdhtml` emits only authored ids, so identical fragments render identically wherever they later appear. `mdhtml2html`'s `auto_ids` option (on by default there) derives an id for each heading without one, and any converter that derives section ids must use the same rules (`gh_ids` below is the one exception): text is lowercased; whitespace becomes `-`; characters other than letters, numbers, `_`, `-`, and `.` are removed; leading nonletters are removed; and an empty result becomes `section`. Duplicate ids receive `-1`, `-2`, and so on, deduplicated within one export. Explicit ids win and participate in duplicate detection. So `## Hello, world!` exports as `

Hello, world!

`. Embedders rendering several fragments into one page pass `auto_ids=False`, since per-fragment derivation cannot see a sibling fragment's ids. + +`mdhtml2html`'s `gh_ids=True` derives those ids by GitHub's rules instead (as `github-slugger` implements them): text is lowercased; letters, numbers, marks, `_`, `-`, and spaces are kept, everything else dropped; each remaining space becomes `-`. It exists because an anchor on a GitHub-rendered page is a published address which the default rules break. Unlike the default rules, the text is read verbatim (untrimmed, uncollapsed, only U+0020 becoming `-`) and there is no `section` fallback: a heading with nothing left gets no id (GitHub's own anchor for it is unaddressable) but holds its dedupe slot, so its repeats take `-1`, `-2` in step with GitHub. Parse options which infer document structure are off by default. Explicit Markdown syntax remains enabled: for example, an explicit heading id is emitted without any option, and bracket math is recognized because its delimiters state the author's intent. `implicit_figures` enables an inferred transformation. diff --git a/python/mdhtml/export.py b/python/mdhtml/export.py index a61f5a5..f2b0b71 100644 --- a/python/mdhtml/export.py +++ b/python/mdhtml/export.py @@ -81,7 +81,7 @@ def go(text, lang): def mdhtml2html(src, dest=None, reftypes: dict | None = None, number_headings=None, hl: str | None = "spans", auto_ids: bool = True, - toc: bool = False, refs: str = "resolve", id_prefix: str = "", fn_salt: str = "", hl_lang=None, code_wrap=None) -> Html: + toc: bool = False, refs: str = "resolve", id_prefix: str = "", fn_salt: str = "", hl_lang=None, code_wrap=None, gh_ids: bool = False) -> Html: """Lower MDHTML (a string or DocumentFragment; never mutated) to finished HTML: cross-references baked as links, headings and captions numbered, `{=html}` raw data spliced, `colwidths` lowered, and code highlighted. A `div` classed `details` lowers to a `
` element, its @@ -90,6 +90,8 @@ def mdhtml2html(src, dest=None, reftypes: dict | None = None, number_headings=No spaces to hyphens, `-1` suffixes on duplicates); pass `auto_ids=False` when rendering fragments that share a page, where per-fragment derived ids would collide. Authored ids (never auto-derived ones) also get a `data-id` attribute, which anchor displays key on. + `gh_ids=True` derives them by GitHub's rules instead (github-slugger's), so anchors match a + GitHub-rendered page and links written against one keep working. `refs='ids'` instead bakes each reference as a working link showing its target id (class `xref`), with no registry, numbering, or failure modes - for live-preview contexts where targets may sit outside the fragment. `refs='lenient'` sits between the two: @@ -108,7 +110,7 @@ def mdhtml2html(src, dest=None, reftypes: dict | None = None, number_headings=No if refs not in ("resolve", "ids", "lenient"): raise ValueError(f"unknown refs mode {refs!r}") if not isinstance(src, str): src = src.to_html() hl_fn = None if hl is None else _hl_fn(hl) - out, warnings = _export_html(src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, hl_fn, auto_ids) + out, warnings = _export_html(src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, hl_fn, auto_ids, gh_ids) res = Html(out, warnings) if dest is not None: Path(dest).write_text(res, encoding="utf-8") return res diff --git a/python/mdhtml/md2html.py b/python/mdhtml/md2html.py index e8e359a..4620721 100644 --- a/python/mdhtml/md2html.py +++ b/python/mdhtml/md2html.py @@ -92,6 +92,7 @@ def main( dark_theme: str = "vscode_dark", # Code colors in dark mode templates: bool = True, # Show mustache `{{tokens}}` as styled pills auto_ids: bool = True, # Derive ids for headings + gh_ids: bool = False, # Derive heading ids by GitHub's rules, matching anchors on a GitHub-rendered page implicit_figures: bool = True, # Promote image-only paragraphs to figures frontmatter: bool = False, # Recognize leading `key: value` frontmatter: strip it, title the page, prepend a metadata table **kwargs @@ -99,7 +100,8 @@ def main( "Read Markdown and write a finished HTML page" tmpl = dict(templates=MUSTACHE, callbacks={'template_token': mustache_pill}) if templates else {} src = md2mdhtml(read_src(file), implicit_figures=implicit_figures, frontmatter=frontmatter, **tmpl, **kwargs) - html = mdhtml2html(src, auto_ids=auto_ids, refs=refs, number_headings=number_headings, toc=toc, hl=None if hl == HlMode.off else hl, code_wrap=_code_wrap) + html = mdhtml2html(src, auto_ids=auto_ids, gh_ids=gh_ids, refs=refs, number_headings=number_headings, toc=toc, + hl=None if hl == HlMode.off else hl, code_wrap=_code_wrap) for w in [*src.warnings, *html.warnings]: print(w, file=sys.stderr) if src.meta: html = meta_table(src.meta) + html title = src.meta.get("title") or (Path(file).stem if file else "mdhtml") diff --git a/src/export_html.rs b/src/export_html.rs index 1d7457f..38cb3f6 100644 --- a/src/export_html.rs +++ b/src/export_html.rs @@ -6,6 +6,7 @@ use std::collections::{HashMap, HashSet}; use fast5ever::{DOCUMENT, Dom, NodeData, NodeId, parse_fragment}; +use unicode_properties::{GeneralCategoryGroup, UnicodeGeneralCategory}; use crate::resolve::{self, HeadingNums, Resolver, target_kind}; @@ -57,6 +58,7 @@ pub struct HtmlExportOptions<'a> { pub code_wrap: Option>, pub hl_fn: Option>, pub auto_ids: bool, + pub gh_ids: bool, } /// Lower an MDHTML fragment to finished HTML; returns the markup and the @@ -136,6 +138,21 @@ fn slug(text: &str) -> String { if out.is_empty() { "section".to_string() } else { out } } +/// Slug for automatic heading ids, GitHub's derivation rules, as +/// `github-slugger` implements them. Its quirks are corpus-verified GitHub +/// behavior, so keep them: only U+0020 hyphenates (other whitespace drops), +/// no trim or collapse, and U+200D survives to hold emoji sequences together. +/// An empty result mints no id (unaddressable, on GitHub too) but holds its +/// dedupe slot, so repeats still take `-1`, `-2` in step with GitHub. +fn slug_github(text: &str) -> String { + let keep = |ch: char| { + ch == '-' + || ch == '_' + || ch == '\u{200D}' + || matches!(ch.general_category_group(), GeneralCategoryGroup::Letter | GeneralCategoryGroup::Number | GeneralCategoryGroup::Mark) + }; + text.to_lowercase().chars().filter(|&c| c == ' ' || keep(c)).map(|c| if c == ' ' { '-' } else { c }).collect() +} impl Exporter { fn run(&mut self, opts: &HtmlExportOptions) -> Result<(), String> { self.lower_details(); @@ -152,7 +169,7 @@ impl Exporter { } } if opts.auto_ids { - self.auto_ids(&els); + self.auto_ids(&els, opts); } for &e in &els { let Some(id) = self.dom.attr(e, "id").map(str::to_string) else { @@ -250,24 +267,27 @@ impl Exporter { } } - /// Pandoc-style ids for headings without one: lowercased, spaces to - /// hyphens, punctuation dropped, leading non-letters stripped, `-1` - /// suffixes on duplicates; explicit ids join duplicate detection and win. - fn auto_ids(&mut self, els: &[NodeId]) { + /// Ids for headings without one: `slug` rules, or `slug_github` on the + /// verbatim text with `gh_ids`, since GitHub keeps leading and doubled + /// spaces significant. `-1` suffixes on duplicates; explicit ids join + /// duplicate detection and win. + fn auto_ids(&mut self, els: &[NodeId], opts: &HtmlExportOptions) { let mut taken: HashSet = els.iter().filter_map(|&e| self.dom.attr(e, "id").map(str::to_string)).collect(); for i in 0..self.heads.len() { let h = self.heads[i]; if self.dom.attr(h, "id").is_some() { continue; } - let base = slug(&norm_text(&self.dom, h)); + let base = if opts.gh_ids { slug_github(&self.dom.to_text(h)) } else { slug(&norm_text(&self.dom, h)) }; let mut id = base.clone(); let mut n = 0; while !taken.insert(id.clone()) { n += 1; id = format!("{base}-{n}"); } - self.dom.set_attr(h, "id", &id).unwrap(); + if !id.is_empty() { + self.dom.set_attr(h, "id", &id).unwrap(); + } } } diff --git a/src/python.rs b/src/python.rs index cdc8db0..2ea156f 100644 --- a/src/python.rs +++ b/src/python.rs @@ -1025,7 +1025,7 @@ fn attr_node<'py>(py: Python<'py>, attrs: &Attr) -> PyResult> // --------------------------------------------------------------------------- #[pyfunction] -#[pyo3(signature = (src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, hl_fn, auto_ids))] +#[pyo3(signature = (src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, hl_fn, auto_ids, gh_ids))] fn export_html( py: Python<'_>, src: &str, @@ -1040,6 +1040,7 @@ fn export_html( code_wrap: Option>, hl_fn: Option>, auto_ids: bool, + gh_ids: bool, ) -> PyResult<(String, Vec)> { use crate::export_html::{HlMode, HtmlExportOptions, NumberHeadings, RefsMode}; let number_headings = match number_headings { @@ -1092,6 +1093,7 @@ fn export_html( code_wrap: code_wrap_c.as_ref().map(|c| c as _), hl_fn: hl_fn_c.as_ref().map(|c| c as _), auto_ids, + gh_ids, }; let result = if hl_lang.is_none() && code_wrap.is_none() && hl_fn.is_none() { py.detach(|| crate::export_html::export_html(src, &opts)) diff --git a/tests/test_export.py b/tests/test_export.py index 3660423..255ef93 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -1,3 +1,5 @@ +import re + import pytest from mdhtml import TemplateDelimiter, dialect_css, math_js, mdhtml2dom, mdhtml2html, md2gfm, md2mdhtml @@ -121,6 +123,33 @@ def test_toc(): assert 'Three' in h.split('')[0] # id-less heading still listed +def ids(md, **kw): return re.findall(r']*\bid="([^"]*)"', mdhtml2html(md2mdhtml(md), **kw)) + + +def test_gh_ids(): + # Each case is a divergence from the default rules, checked against GitHub's own anchor. + assert ids('# Footnotes.') == ['footnotes.'] # default keeps the period + assert ids('# Footnotes.', gh_ids=True) == ['footnotes'] # GitHub drops it + assert ids('# 2021-03-16') == ['section'] # no letter to start from + assert ids('# 2021-03-16', gh_ids=True) == ['2021-03-16'] # digits are kept + assert ids('# --page-file-dir', gh_ids=True) == ['--page-file-dir'] + assert ids('# Using custom.css', gh_ids=True) == ['using-customcss'] + assert ids('# Minutes\n*', gh_ids=True) == ['minutes'] # a newline is dropped, not hyphenated + assert ids('# ![](i.png) Wiki', gh_ids=True) == ['-wiki'] # not trimmed: the image leaves a leading space + assert ids('# A B', gh_ids=True) == ['a--b'] # not collapsed + # Emoji go, but ZWJ and variation selector inside a sequence stay, as GitHub's list omits them. + assert ids('# \U0001f477\u200d\u2640\ufe0f Projects', gh_ids=True) == ['\u200d\ufe0f-projects'] + + +def test_gh_ids_dedup(): + assert ids('# Repeat\n\n# Repeat\n\n# Repeat', gh_ids=True) == ['repeat', 'repeat-1', 'repeat-2'] + assert ids('# Repeat {#repeat}\n\n# Repeat', gh_ids=True) == ['repeat', 'repeat-1'] # authored id wins, joins dedupe + # No 'section' fallback: an empty slug mints no id (id="" is invalid HTML and unaddressable, + # on GitHub too) but holds its dedupe slot, so the next repeat is '-1', matching GitHub. + assert ids('# ***\n\n# ***', gh_ids=True) == ['-1'] + assert ids('# Hello', gh_ids=True, auto_ids=False) == [] # gh_ids mints nothing alone + + def test_api_shape(tmp_path): frag = mdhtml2dom('

Hi

') before = frag.to_html()