Add gh_ids: derive heading ids by GitHub's rules

docs/DIALECT.md

@@ -95,7 +95,9 @@ Headings use `h1` through `h6`; paragraphs, thematic breaks, and block quotes us
 
 ## Identifiers and rendering options
 
-Automatic heading ids are an export concern, not part of the parse: `md2mdhtml` emits only authored ids, so identical fragments render identically wherever they later appear. `mdhtml2html`'s `auto_ids` option (on by default there) derives an id for each heading without one, and any converter that derives section ids must use the same rules: text is lowercased; whitespace becomes `-`; characters other than letters, numbers, `_`, `-`, and `.` are removed; leading nonletters are removed; and an empty result becomes `section`. Duplicate ids receive `-1`, `-2`, and so on, deduplicated within one export. Explicit ids win and participate in duplicate detection. So `## Hello, world!` exports as `<h2 id="hello-world">Hello, world!</h2>`. Embedders rendering several fragments into one page pass `auto_ids=False`, since per-fragment derivation cannot see a sibling fragment's ids.
+Automatic heading ids are an export concern, not part of the parse: `md2mdhtml` emits only authored ids, so identical fragments render identically wherever they later appear. `mdhtml2html`'s `auto_ids` option (on by default there) derives an id for each heading without one, and any converter that derives section ids must use the same rules (`gh_ids` below is the one exception): text is lowercased; whitespace becomes `-`; characters other than letters, numbers, `_`, `-`, and `.` are removed; leading nonletters are removed; and an empty result becomes `section`. Duplicate ids receive `-1`, `-2`, and so on, deduplicated within one export. Explicit ids win and participate in duplicate detection. So `## Hello, world!` exports as `<h2 id="hello-world">Hello, world!</h2>`. Embedders rendering several fragments into one page pass `auto_ids=False`, since per-fragment derivation cannot see a sibling fragment's ids.
+
+`mdhtml2html`'s `gh_ids=True` derives those ids by GitHub's rules instead (as `github-slugger` implements them): text is lowercased; letters, numbers, marks, `_`, `-`, and spaces are kept, everything else dropped; each remaining space becomes `-`. It exists because an anchor on a GitHub-rendered page is a published address which the default rules break. Unlike the default rules, the text is read verbatim (untrimmed, uncollapsed, only U+0020 becoming `-`) and there is no `section` fallback: a heading with nothing left gets no id (GitHub's own anchor for it is unaddressable) but holds its dedupe slot, so its repeats take `-1`, `-2` in step with GitHub.
 
 Parse options which infer document structure are off by default. Explicit Markdown syntax remains enabled: for example, an explicit heading id is emitted without any option, and bracket math is recognized because its delimiters state the author's intent. `implicit_figures` enables an inferred transformation.
 

python/mdhtml/export.py

@@ -81,7 +81,7 @@ def go(text, lang):
 
 
 def mdhtml2html(src, dest=None, reftypes: dict | None = None, number_headings=None, hl: str | None = "spans", auto_ids: bool = True,
-    toc: bool = False, refs: str = "resolve", id_prefix: str = "", fn_salt: str = "", hl_lang=None, code_wrap=None) -> Html:
+    toc: bool = False, refs: str = "resolve", id_prefix: str = "", fn_salt: str = "", hl_lang=None, code_wrap=None, gh_ids: bool = False) -> Html:
     """Lower MDHTML (a string or DocumentFragment; never mutated) to finished HTML: cross-references
     baked as links, headings and captions numbered, `{=html}` raw data spliced, `colwidths` lowered,
     and code highlighted. A `div` classed `details` lowers to a `<details>` element, its
@@ -90,6 +90,8 @@ def mdhtml2html(src, dest=None, reftypes: dict | None = None, number_headings=No
     spaces to hyphens, `-1` suffixes on duplicates); pass `auto_ids=False` when rendering fragments
     that share a page, where per-fragment derived ids would collide. Authored ids (never
     auto-derived ones) also get a `data-id` attribute, which anchor displays key on.
+    `gh_ids=True` derives them by GitHub's rules instead (github-slugger's), so anchors match a
+    GitHub-rendered page and links written against one keep working.
     `refs='ids'` instead bakes each reference as a working link showing its
     target id (class `xref`), with no registry, numbering, or failure modes - for live-preview
     contexts where targets may sit outside the fragment. `refs='lenient'` sits between the two:
@@ -108,7 +110,7 @@ def mdhtml2html(src, dest=None, reftypes: dict | None = None, number_headings=No
     if refs not in ("resolve", "ids", "lenient"): raise ValueError(f"unknown refs mode {refs!r}")
     if not isinstance(src, str): src = src.to_html()
     hl_fn = None if hl is None else _hl_fn(hl)
-    out, warnings = _export_html(src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, hl_fn, auto_ids)
+    out, warnings = _export_html(src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, hl_fn, auto_ids, gh_ids)
     res = Html(out, warnings)
     if dest is not None: Path(dest).write_text(res, encoding="utf-8")
     return res

python/mdhtml/md2html.py

@@ -92,14 +92,16 @@ def main(
     dark_theme: str = "vscode_dark",  # Code colors in dark mode
     templates: bool = True,  # Show mustache `{{tokens}}` as styled pills
     auto_ids: bool = True,  # Derive ids for headings
+    gh_ids: bool = False,  # Derive heading ids by GitHub's rules, matching anchors on a GitHub-rendered page
     implicit_figures: bool = True,  # Promote image-only paragraphs to figures
     frontmatter: bool = False,  # Recognize leading `key: value` frontmatter: strip it, title the page, prepend a metadata table
     **kwargs
 ):
     "Read Markdown and write a finished HTML page"
     tmpl = dict(templates=MUSTACHE, callbacks={'template_token': mustache_pill}) if templates else {}
     src = md2mdhtml(read_src(file), implicit_figures=implicit_figures, frontmatter=frontmatter, **tmpl, **kwargs)
-    html = mdhtml2html(src, auto_ids=auto_ids, refs=refs, number_headings=number_headings, toc=toc, hl=None if hl == HlMode.off else hl, code_wrap=_code_wrap)
+    html = mdhtml2html(src, auto_ids=auto_ids, gh_ids=gh_ids, refs=refs, number_headings=number_headings, toc=toc,
+        hl=None if hl == HlMode.off else hl, code_wrap=_code_wrap)
     for w in [*src.warnings, *html.warnings]: print(w, file=sys.stderr)
     if src.meta: html = meta_table(src.meta) + html
     title = src.meta.get("title") or (Path(file).stem if file else "mdhtml")

src/export_html.rs

@@ -6,6 +6,7 @@
 use std::collections::{HashMap, HashSet};
 
 use fast5ever::{DOCUMENT, Dom, NodeData, NodeId, parse_fragment};
+use unicode_properties::{GeneralCategoryGroup, UnicodeGeneralCategory};
 
 use crate::resolve::{self, HeadingNums, Resolver, target_kind};
 
@@ -57,6 +58,7 @@ pub struct HtmlExportOptions<'a> {
     pub code_wrap: Option<CodeWrapHook<'a>>,
     pub hl_fn: Option<HlHook<'a>>,
     pub auto_ids: bool,
+    pub gh_ids: bool,
 }
 
 /// Lower an MDHTML fragment to finished HTML; returns the markup and the
@@ -136,6 +138,21 @@ fn slug(text: &str) -> String {
     if out.is_empty() { "section".to_string() } else { out }
 }
 
+/// Slug for automatic heading ids, GitHub's derivation rules, as
+/// `github-slugger` implements them. Its quirks are corpus-verified GitHub
+/// behavior, so keep them: only U+0020 hyphenates (other whitespace drops),
+/// no trim or collapse, and U+200D survives to hold emoji sequences together.
+/// An empty result mints no id (unaddressable, on GitHub too) but holds its
+/// dedupe slot, so repeats still take `-1`, `-2` in step with GitHub.
+fn slug_github(text: &str) -> String {
+    let keep = |ch: char| {
+        ch == '-'
+            || ch == '_'
+            || ch == '\u{200D}'
+            || matches!(ch.general_category_group(), GeneralCategoryGroup::Letter | GeneralCategoryGroup::Number | GeneralCategoryGroup::Mark)
+    };
+    text.to_lowercase().chars().filter(|&c| c == ' ' || keep(c)).map(|c| if c == ' ' { '-' } else { c }).collect()
+}
 impl Exporter {
     fn run(&mut self, opts: &HtmlExportOptions) -> Result<(), String> {
         self.lower_details();
@@ -152,7 +169,7 @@ impl Exporter {
             }
         }
         if opts.auto_ids {
-            self.auto_ids(&els);
+            self.auto_ids(&els, opts);
         }
         for &e in &els {
             let Some(id) = self.dom.attr(e, "id").map(str::to_string) else {
@@ -250,24 +267,27 @@ impl Exporter {
         }
     }
 
-    /// Pandoc-style ids for headings without one: lowercased, spaces to
-    /// hyphens, punctuation dropped, leading non-letters stripped, `-1`
-    /// suffixes on duplicates; explicit ids join duplicate detection and win.
-    fn auto_ids(&mut self, els: &[NodeId]) {
+    /// Ids for headings without one: `slug` rules, or `slug_github` on the
+    /// verbatim text with `gh_ids`, since GitHub keeps leading and doubled
+    /// spaces significant. `-1` suffixes on duplicates; explicit ids join
+    /// duplicate detection and win.
+    fn auto_ids(&mut self, els: &[NodeId], opts: &HtmlExportOptions) {
         let mut taken: HashSet<String> = els.iter().filter_map(|&e| self.dom.attr(e, "id").map(str::to_string)).collect();
         for i in 0..self.heads.len() {
             let h = self.heads[i];
             if self.dom.attr(h, "id").is_some() {
                 continue;
             }
-            let base = slug(&norm_text(&self.dom, h));
+            let base = if opts.gh_ids { slug_github(&self.dom.to_text(h)) } else { slug(&norm_text(&self.dom, h)) };
             let mut id = base.clone();
             let mut n = 0;
             while !taken.insert(id.clone()) {
                 n += 1;
                 id = format!("{base}-{n}");
             }
-            self.dom.set_attr(h, "id", &id).unwrap();
+            if !id.is_empty() {
+                self.dom.set_attr(h, "id", &id).unwrap();
+            }
         }
     }
 

src/python.rs

@@ -1025,7 +1025,7 @@ fn attr_node<'py>(py: Python<'py>, attrs: &Attr) -> PyResult<Bound<'py, PyDict>>
 // ---------------------------------------------------------------------------
 
 #[pyfunction]
-#[pyo3(signature = (src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, hl_fn, auto_ids))]
+#[pyo3(signature = (src, reftypes, number_headings, hl, toc, refs, id_prefix, fn_salt, hl_lang, code_wrap, hl_fn, auto_ids, gh_ids))]
 fn export_html(
     py: Python<'_>,
     src: &str,
@@ -1040,6 +1040,7 @@ fn export_html(
     code_wrap: Option<Py<PyAny>>,
     hl_fn: Option<Py<PyAny>>,
     auto_ids: bool,
+    gh_ids: bool,
 ) -> PyResult<(String, Vec<String>)> {
     use crate::export_html::{HlMode, HtmlExportOptions, NumberHeadings, RefsMode};
     let number_headings = match number_headings {
@@ -1092,6 +1093,7 @@ fn export_html(
         code_wrap: code_wrap_c.as_ref().map(|c| c as _),
         hl_fn: hl_fn_c.as_ref().map(|c| c as _),
         auto_ids,
+        gh_ids,
     };
     let result = if hl_lang.is_none() && code_wrap.is_none() && hl_fn.is_none() {
         py.detach(|| crate::export_html::export_html(src, &opts))

tests/test_export.py

@@ -1,3 +1,5 @@
+import re
+
 import pytest
 
 from mdhtml import TemplateDelimiter, dialect_css, math_js, mdhtml2dom, mdhtml2html, md2gfm, md2mdhtml
@@ -121,6 +123,33 @@ def test_toc():
     assert 'Three' in h.split('</nav>')[0]                         # id-less heading still listed
 
 
+def ids(md, **kw): return re.findall(r'<h[1-6][^>]*\bid="([^"]*)"', mdhtml2html(md2mdhtml(md), **kw))
+
+
+def test_gh_ids():
+    # Each case is a divergence from the default rules, checked against GitHub's own anchor.
+    assert ids('# Footnotes.') == ['footnotes.']                    # default keeps the period
+    assert ids('# Footnotes.', gh_ids=True) == ['footnotes']        # GitHub drops it
+    assert ids('# 2021-03-16') == ['section']                       # no letter to start from
+    assert ids('# 2021-03-16', gh_ids=True) == ['2021-03-16']       # digits are kept
+    assert ids('# --page-file-dir', gh_ids=True) == ['--page-file-dir']
+    assert ids('# Using custom.css', gh_ids=True) == ['using-customcss']
+    assert ids('# Minutes\n*', gh_ids=True) == ['minutes']          # a newline is dropped, not hyphenated
+    assert ids('# ![](i.png) Wiki', gh_ids=True) == ['-wiki']       # not trimmed: the image leaves a leading space
+    assert ids('# A  B', gh_ids=True) == ['a--b']                   # not collapsed
+    # Emoji go, but ZWJ and variation selector inside a sequence stay, as GitHub's list omits them.
+    assert ids('# \U0001f477\u200d\u2640\ufe0f Projects', gh_ids=True) == ['\u200d\ufe0f-projects']
+
+
+def test_gh_ids_dedup():
+    assert ids('# Repeat\n\n# Repeat\n\n# Repeat', gh_ids=True) == ['repeat', 'repeat-1', 'repeat-2']
+    assert ids('# Repeat {#repeat}\n\n# Repeat', gh_ids=True) == ['repeat', 'repeat-1']   # authored id wins, joins dedupe
+    # No 'section' fallback: an empty slug mints no id (id="" is invalid HTML and unaddressable,
+    # on GitHub too) but holds its dedupe slot, so the next repeat is '-1', matching GitHub.
+    assert ids('# ***\n\n# ***', gh_ids=True) == ['-1']
+    assert ids('# Hello', gh_ids=True, auto_ids=False) == []        # gh_ids mints nothing alone
+
+
 def test_api_shape(tmp_path):
     frag = mdhtml2dom('<p id="x">Hi</p><p><a data-ref="bare text" href="#x"></a></p>')
     before = frag.to_html()