From 363741c5a84796513d215c835004b954268ed6a8 Mon Sep 17 00:00:00 2001 From: fadeltd Date: Fri, 25 Sep 2026 13:06:06 +0700 Subject: [PATCH] Add HTML to Text tool A pure tokenizer in core/ (no DOM, no new dependency) that keeps paragraph, list and table structure, drops script/style/head/svg, hidden and role=img subtrees, and decodes entities including the windows-1252 numeric remap. Optional link URLs and class-based hidden heuristics. --- src/lib/registry/index.ts | 26 +- src/tools/html-text/HtmlTextTool.tsx | 99 ++++ src/tools/html-text/core/entities.ts | 168 +++++++ src/tools/html-text/core/extract.test.ts | 279 +++++++++++ src/tools/html-text/core/extract.ts | 465 ++++++++++++++++++ .../html-text/core/fixture-notification.html | 1 + 6 files changed, 1037 insertions(+), 1 deletion(-) create mode 100644 src/tools/html-text/HtmlTextTool.tsx create mode 100644 src/tools/html-text/core/entities.ts create mode 100644 src/tools/html-text/core/extract.test.ts create mode 100644 src/tools/html-text/core/extract.ts create mode 100644 src/tools/html-text/core/fixture-notification.html diff --git a/src/lib/registry/index.ts b/src/lib/registry/index.ts index d86d91a..2ed02f1 100644 --- a/src/lib/registry/index.ts +++ b/src/lib/registry/index.ts @@ -1,4 +1,4 @@ -import { Binary, Braces, GitCompare, KeyRound, ListOrdered, Sigma } from 'lucide-react' +import { Binary, Braces, CodeXml, GitCompare, KeyRound, ListOrdered, Sigma } from 'lucide-react' import type { ToolDef } from './types' /** @@ -127,6 +127,30 @@ const RAW_TOOLS = [ stateVersion: 1, status: 'beta', }, + { + slug: 'html-text', + title: 'HTML to Text', + blurb: 'Extract readable plain text from HTML, keeping paragraphs, lists and tables intact.', + keywords: [ + 'html to text', + 'extract text', + 'strip html', + 'strip tags', + 'remove html tags', + 'html stripper', + 'plain text', + 'innertext', + 'textcontent', + 'html2text', + ], + category: 'web', + icon: CodeXml, + load: () => import('@/tools/html-text/HtmlTextTool'), + // Pasted page source runs to several MB: too big for localStorage. + store: { kind: 'idb' }, + stateVersion: 1, + status: 'beta', + }, ] as const satisfies readonly ToolDef[] /** The literal union that keeps persistence keys, links and pins honest. */ diff --git a/src/tools/html-text/HtmlTextTool.tsx b/src/tools/html-text/HtmlTextTool.tsx new file mode 100644 index 0000000..6a9baf8 --- /dev/null +++ b/src/tools/html-text/HtmlTextTool.tsx @@ -0,0 +1,99 @@ +import { useDeferredValue, useMemo } from 'react' +import { ToolFrame } from '@/components/layout/ToolFrame' +import { TwoPane } from '@/components/layout/TwoPane' +import { Button } from '@/components/ui/Button' +import { CodeArea } from '@/components/ui/CodeArea' +import { CopyButton } from '@/components/ui/CopyButton' +import { Toggle } from '@/components/ui/Select' +import { useToolState } from '@/lib/persist/useToolState' +import { useShortcuts } from '@/lib/keys/useShortcuts' +import { useToolUsageTracker } from '@/lib/prefs' +import { copyText } from '@/lib/util/clipboard' +import { formatBytes } from '@/lib/util/bytes' +import { htmlToText } from './core/extract' + +interface State { + html: string + linkUrls: boolean + skipHiddenClasses: boolean +} + +const INITIAL: State = { html: '', linkUrls: false, skipHiddenClasses: false } + +export default function HtmlTextTool() { + useToolUsageTracker('html-text') + const { state, setState, reset } = useToolState('html-text', INITIAL) + + const deferred = useDeferredValue(state) + const { output, sizeNote } = useMemo(() => { + if (deferred.html === '') return { output: '', sizeNote: null } + const text = htmlToText(deferred.html, { + linkUrls: deferred.linkUrls, + skipHiddenClasses: deferred.skipHiddenClasses, + }) + const enc = new TextEncoder() + return { + output: text, + sizeNote: `${formatBytes(enc.encode(deferred.html).length)} → ${formatBytes(enc.encode(text).length)}`, + } + }, [deferred]) + + useShortcuts([ + { + combo: 'mod+shift+c', + label: 'Copy output', + group: 'HTML to Text', + scope: 'tool', + whileTyping: true, + run: () => void copyText(output), + }, + ]) + + return ( + + setState((p) => ({ ...p, linkUrls: v }))} + > + Show link URLs + + setState((p) => ({ ...p, skipHiddenClasses: v }))} + > + Skip .hidden / .sr-only + + + + + } + > + setState((p) => ({ ...p, html: e.target.value }))} + placeholder="Paste HTML — a page source, an email, or a copied element…" + /> + } + right={ +
+ + {sizeNote !== null && ( +
+ {sizeNote} +
+ )} +
+ } + /> +
+ ) +} diff --git a/src/tools/html-text/core/entities.ts b/src/tools/html-text/core/entities.ts new file mode 100644 index 0000000..e23c3b0 --- /dev/null +++ b/src/tools/html-text/core/entities.ts @@ -0,0 +1,168 @@ +/** + * Named character references, deliberately partial. The full HTML5 table is + * ~2,200 entries (~10 kB gzip) for a tail nobody pastes; an unknown name is + * left untouched rather than guessed at, so a miss is visible, never corrupt. + */ +const NAMED: Record = { + amp: 0x26, + lt: 0x3c, + gt: 0x3e, + quot: 0x22, + apos: 0x27, + OElig: 0x152, + oelig: 0x153, + Scaron: 0x160, + scaron: 0x161, + Yuml: 0x178, + fnof: 0x192, + circ: 0x2c6, + tilde: 0x2dc, + Delta: 0x394, + Pi: 0x3a0, + Sigma: 0x3a3, + Omega: 0x3a9, + alpha: 0x3b1, + beta: 0x3b2, + gamma: 0x3b3, + delta: 0x3b4, + epsilon: 0x3b5, + theta: 0x3b8, + lambda: 0x3bb, + mu: 0x3bc, + pi: 0x3c0, + sigma: 0x3c3, + tau: 0x3c4, + phi: 0x3c6, + omega: 0x3c9, + ensp: 0x2002, + emsp: 0x2003, + thinsp: 0x2009, + zwnj: 0x200c, + zwj: 0x200d, + lrm: 0x200e, + rlm: 0x200f, + ndash: 0x2013, + mdash: 0x2014, + lsquo: 0x2018, + rsquo: 0x2019, + sbquo: 0x201a, + ldquo: 0x201c, + rdquo: 0x201d, + bdquo: 0x201e, + dagger: 0x2020, + Dagger: 0x2021, + bull: 0x2022, + hellip: 0x2026, + permil: 0x2030, + prime: 0x2032, + Prime: 0x2033, + lsaquo: 0x2039, + rsaquo: 0x203a, + euro: 0x20ac, + trade: 0x2122, + larr: 0x2190, + uarr: 0x2191, + rarr: 0x2192, + darr: 0x2193, + harr: 0x2194, + crarr: 0x21b5, + lArr: 0x21d0, + rArr: 0x21d2, + hArr: 0x21d4, + forall: 0x2200, + part: 0x2202, + exist: 0x2203, + empty: 0x2205, + nabla: 0x2207, + isin: 0x2208, + notin: 0x2209, + prod: 0x220f, + sum: 0x2211, + minus: 0x2212, + radic: 0x221a, + infin: 0x221e, + and: 0x2227, + or: 0x2228, + cap: 0x2229, + cup: 0x222a, + there4: 0x2234, + asymp: 0x2248, + ne: 0x2260, + equiv: 0x2261, + le: 0x2264, + ge: 0x2265, + loz: 0x25ca, + spades: 0x2660, + clubs: 0x2663, + hearts: 0x2665, + diams: 0x2666, +} + +// U+00A0..U+00FF all have names, in code point order. +const LATIN1 = + 'nbsp iexcl cent pound curren yen brvbar sect uml copy ordf laquo not shy reg macr ' + + 'deg plusmn sup2 sup3 acute micro para middot cedil sup1 ordm raquo frac14 frac12 frac34 iquest ' + + 'Agrave Aacute Acirc Atilde Auml Aring AElig Ccedil Egrave Eacute Ecirc Euml Igrave Iacute Icirc Iuml ' + + 'ETH Ntilde Ograve Oacute Ocirc Otilde Ouml times Oslash Ugrave Uacute Ucirc Uuml Yacute THORN szlig ' + + 'agrave aacute acirc atilde auml aring aelig ccedil egrave eacute ecirc euml igrave iacute icirc iuml ' + + 'eth ntilde ograve oacute ocirc otilde ouml divide oslash ugrave uacute ucirc uuml yacute thorn yuml' +LATIN1.split(' ').forEach((name, i) => { + NAMED[name] = 0xa0 + i +}) + +/** + * The HTML spec remaps numeric references in 0x80..0x9F to their windows-1252 + * meaning, because that is what `–` always meant in practice. Without it + * an en dash decodes to an invisible C1 control. + */ +const CP1252: Record = { + 0x80: 0x20ac, + 0x82: 0x201a, + 0x83: 0x192, + 0x84: 0x201e, + 0x85: 0x2026, + 0x86: 0x2020, + 0x87: 0x2021, + 0x88: 0x2c6, + 0x89: 0x2030, + 0x8a: 0x160, + 0x8b: 0x2039, + 0x8c: 0x152, + 0x8e: 0x17d, + 0x91: 0x2018, + 0x92: 0x2019, + 0x93: 0x201c, + 0x94: 0x201d, + 0x95: 0x2022, + 0x96: 0x2013, + 0x97: 0x2014, + 0x98: 0x2dc, + 0x99: 0x2122, + 0x9a: 0x161, + 0x9b: 0x203a, + 0x9c: 0x153, + 0x9e: 0x17e, + 0x9f: 0x178, +} + +const ENTITY = /&(?:#(\d+)|#[xX]([0-9a-fA-F]+)|([a-zA-Z][a-zA-Z0-9]*));/g + +function fromCodePoint(cp: number): string { + if (cp === 0 || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff)) return '�' + return String.fromCodePoint(CP1252[cp] ?? cp) +} + +/** + * Decode character references in a single pass, so `&lt;` becomes `<` + * and not `<`. Only the terminated form is recognised: `©=2` in a pasted + * URL stays as typed. + */ +export function decodeEntities(s: string): string { + if (!s.includes('&')) return s + return s.replace(ENTITY, (whole, dec?: string, hex?: string, name?: string) => { + if (dec !== undefined) return fromCodePoint(Number.parseInt(dec, 10)) + if (hex !== undefined) return fromCodePoint(Number.parseInt(hex, 16)) + const cp = Object.hasOwn(NAMED, name!) ? NAMED[name!] : undefined + return cp === undefined ? whole : String.fromCodePoint(cp) + }) +} diff --git a/src/tools/html-text/core/extract.test.ts b/src/tools/html-text/core/extract.test.ts new file mode 100644 index 0000000..624fed4 --- /dev/null +++ b/src/tools/html-text/core/extract.test.ts @@ -0,0 +1,279 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { decodeEntities, htmlToText } from './extract' + +describe('decodeEntities', () => { + it('decodes common named entities', () => { + expect(decodeEntities('a & b <c> "d" 'e'')).toBe(`a & b "d" 'e'`) + }) + + it('decodes nbsp to U+00A0, not a plain space', () => { + expect(decodeEntities('a b')).toBe('a\u00a0b') + }) + + it('decodes decimal and hex numeric references', () => { + expect(decodeEntities('— — 👍')).toBe('— — 👍') + }) + + it('remaps 0x80..0x9F references to windows-1252, as browsers do', () => { + expect(decodeEntities('– “x” €')).toBe('– “x” €') + }) + + it('maps invalid code points to U+FFFD instead of throwing', () => { + expect(decodeEntities('� � �')).toBe('� � �') + }) + + it('leaves unknown named entities untouched', () => { + expect(decodeEntities('&bogus; &')).toBe('&bogus; &') + }) + + it('does not double-decode', () => { + expect(decodeEntities('&lt;')).toBe('<') + }) +}) + +describe('htmlToText: whitespace', () => { + it('returns empty for empty input', () => { + expect(htmlToText('')).toBe('') + }) + + it('passes plain text through', () => { + expect(htmlToText('hello world')).toBe('hello world') + }) + + it('collapses whitespace runs the way a browser does', () => { + expect(htmlToText(' a \n\t b c ')).toBe('a b c') + }) + + it('does not insert spaces between adjacent inline elements', () => { + expect(htmlToText('bold')).toBe('bold') + }) + + it('preserves whitespace inside
', () => {
+    expect(htmlToText('

x

  a\n    b

y

')).toBe('x\n\n a\n b\n\ny') + }) + + it('drops the single newline directly after
', () => {
+    expect(htmlToText('
\nline
')).toBe('line') + }) + + it('turns
into a newline', () => { + expect(htmlToText('a
b
c')).toBe('a\nb\nc') + }) + + it('keeps nbsp as a real space in the output', () => { + expect(htmlToText('a  b')).toBe('a b') + }) + + it('never emits a trailing newline or trailing spaces', () => { + expect(htmlToText('

a

\n\n
b
\n')).toBe('a\n\nb') + }) +}) + +describe('htmlToText: blocks', () => { + it('puts generic blocks on their own lines', () => { + expect(htmlToText('
a
b
')).toBe('a\nb') + }) + + it('separates paragraphs with a blank line', () => { + expect(htmlToText('

a

b

')).toBe('a\n\nb') + }) + + it('separates headings with a blank line', () => { + expect(htmlToText('

Title

text')).toBe('Title\n\ntext') + }) + + it('never produces more than one blank line in a row', () => { + expect(htmlToText('

a

b

')).toBe('a\n\nb') + }) + + it('is case-insensitive about tag names', () => { + expect(htmlToText('

a

b
')).toBe('a\n\nb') + }) +}) + +describe('htmlToText: lists and tables', () => { + it('bullets unordered list items', () => { + expect(htmlToText('
  • a
  • b
')).toBe('- a\n- b') + }) + + it('numbers ordered list items', () => { + expect(htmlToText('
  1. a
  2. b
')).toBe('1. a\n2. b') + }) + + it('respects
    ', () => { + expect(htmlToText('
    1. a
    2. b
    ')).toBe('3. a\n4. b') + }) + + it('indents nested lists', () => { + expect(htmlToText('
    • a
      • b
    • c
    ')).toBe( + '- a\n - b\n- c', + ) + }) + + it('handles implicitly closed
  1. ', () => { + expect(htmlToText('
    • a
    • b
    ')).toBe('- a\n- b') + }) + + it('joins table cells with tabs and rows with newlines', () => { + expect( + htmlToText('
    kv
    a1
    '), + ).toBe('k\tv\na\t1') + }) +}) + +describe('htmlToText: skipped content', () => { + it('drops script, style, template, noscript, head and svg contents', () => { + const html = + 'T' + + '' + + 'skept' + expect(htmlToText(html)).toBe('kept') + }) + + it('does not end a script at a nested tag that looks like a close', () => { + expect(htmlToText('ok')).toBe('ok') + }) + + it('drops comments, including empty Angular markers', () => { + expect(htmlToText('abd')).toBe('abd') + }) + + it('drops doctype and processing instructions', () => { + expect(htmlToText('x')).toBe('x') + }) + + it('drops the text of role="img" elements (icon ligatures)', () => { + expect(htmlToText('Open expand_less')).toBe( + 'Open', + ) + }) + + it('drops elements with the hidden attribute', () => { + expect(htmlToText('
    a
    c
    ')).toBe('a\nc') + }) + + it('drops elements with inline display:none', () => { + expect(htmlToText('abc')).toBe('ac') + }) + + it('does not drop aria-hidden elements (they are often visible)', () => { + expect(htmlToText('')).toBe('visible') + }) + + it('keeps class-hidden elements by default', () => { + expect(htmlToText('')).toBe('b') + }) + + it('drops class-hidden elements when asked', () => { + const html = '
    a
    b
    cd' + expect(htmlToText(html, { skipHiddenClasses: true })).toBe('a\nd') + }) + + it('matches hidden classes as whole words only', () => { + expect(htmlToText('', { skipHiddenClasses: true })).toBe( + 'a', + ) + }) + + it('drops void elements inside a hidden subtree without desyncing', () => { + expect(htmlToText('kept')).toBe('kept') + }) +}) + +describe('htmlToText: links', () => { + it('shows link text only by default', () => { + expect(htmlToText('see docs.')).toBe('see docs.') + }) + + it('appends the URL when asked', () => { + expect(htmlToText('see docs.', { linkUrls: true })).toBe( + 'see docs .', + ) + }) + + it('does not repeat a URL that is already the link text', () => { + expect(htmlToText('https://x.test/', { linkUrls: true })).toBe( + 'https://x.test/', + ) + }) + + it('ignores fragment and javascript: hrefs', () => { + expect( + htmlToText('up go', { linkUrls: true }), + ).toBe('up go') + }) +}) + +describe('htmlToText: malformed input', () => { + it('treats a stray < as text', () => { + expect(htmlToText('1 < 2 and 3<4')).toBe('1 < 2 and 3<4') + }) + + it('tolerates an unterminated tag', () => { + expect(htmlToText('a
    b">x')).toBe('x') + }) + + it('decodes entities in text', () => { + expect(htmlToText('

    Tom & Jerry…

    ')).toBe('Tom & Jerry…') + }) + + it('never throws and never returns tag syntax for well-formed markup', () => { + fc.assert( + fc.property(fc.string(), (s) => { + const out = htmlToText(s) + expect(typeof out).toBe('string') + }), + ) + fc.assert( + fc.property(fc.stringMatching(/^[a-z ]*$/), (s) => { + expect(htmlToText(`

    ${s}

    `)).not.toMatch(/[<>]/) + }), + ) + }) +}) + +describe('htmlToText: real-world fixture', () => { + it('extracts a Play Console notification', () => { + const html = readFileSync(new URL('./fixture-notification.html', import.meta.url), 'utf8') + expect(htmlToText(html)).toBe( + [ + 'New', + 'Headroom: Volume Booster EQ', + 'Sep 23', + 'An SDK version you are using is outdated', + '', + 'androidx.fragment:fragment has reported fragment:1.1.0 as outdated. Consider updating to a newer SDK version.', + '', + 'Affected app bundles and APKs:', + '', + '- version: 2 (0.2.0), release: 0.2.0', + '', + 'If you have questions about this SDK, contact the SDK provider.', + '', + 'Learn more', + ].join('\n'), + ) + }) + + it('drops the hidden button row when class heuristics are on', () => { + const html = readFileSync(new URL('./fixture-notification.html', import.meta.url), 'utf8') + const out = htmlToText(html, { skipHiddenClasses: true }) + expect(out).not.toContain('Learn more') + expect(out).not.toContain('New') + expect(out.startsWith('Headroom: Volume Booster EQ\nSep 23\n')).toBe(true) + }) +}) diff --git a/src/tools/html-text/core/extract.ts b/src/tools/html-text/core/extract.ts new file mode 100644 index 0000000..c4b334e --- /dev/null +++ b/src/tools/html-text/core/extract.ts @@ -0,0 +1,465 @@ +import { decodeEntities } from './entities' + +export { decodeEntities } + +export interface ExtractOptions { + /** Append `` after link text. */ + linkUrls: boolean + /** + * Also drop elements whose class *looks* hidden (`hidden`, `sr-only`, ...). + * A naming-convention guess, so it is opt-in. + */ + skipHiddenClasses: boolean +} + +export const DEFAULT_OPTIONS: ExtractOptions = { linkUrls: false, skipHiddenClasses: false } + +const VOID = new Set([ + 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', + 'track', 'wbr', +]) + +/** Content is raw text up to the matching close tag; never parsed as markup. */ +const RAW_TEXT = new Set([ + 'script', 'style', 'textarea', 'title', 'xmp', 'iframe', 'noembed', 'noframes', 'noscript', +]) + +/** Never rendered as text, whatever is inside. */ +const SKIP = new Set([ + 'head', 'script', 'style', 'title', 'template', 'noscript', 'iframe', 'noembed', 'noframes', + 'svg', 'math', 'canvas', 'audio', 'video', 'object', +]) + +/** Blocks separated by a blank line. `ul`/`ol` are here only at the top level. */ +const PARAGRAPH = new Set([ + 'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'pre', 'listing', 'table', 'dl', 'ul', + 'ol', 'menu', +]) + +/** Blocks that start on their own line. Unknown elements are inline, as in a browser. */ +const LINE = new Set([ + 'address', 'article', 'aside', 'body', 'caption', 'center', 'dd', 'details', 'dialog', 'div', + 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'header', 'hgroup', 'hr', 'legend', + 'li', 'main', 'nav', 'option', 'section', 'summary', 'tr', +]) + +/** Opening one of these closes an open `

    `, as the HTML parser does. */ +const CLOSES_P = new Set([ + 'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'div', 'dl', 'fieldset', + 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', + 'hr', 'li', 'main', 'menu', 'nav', 'ol', 'p', 'pre', 'section', 'table', 'ul', +]) + +/** An implicit close never reaches past these. */ +const SCOPE = new Set(['table', 'td', 'th', 'caption', 'template', 'button', 'object']) + +const HIDDEN_CLASSES = new Set([ + 'hidden', 'hide', 'is-hidden', 'sr-only', 'visually-hidden', 'visuallyhidden', 'screen-reader-text', + 'd-none', 'invisible', +]) + +const LISTS = new Set(['ul', 'ol', 'menu']) + +/** Implicit-close rules: [what a start tag closes, where the search stops]. */ +const IMPLICIT: Record, ReadonlySet]> = { + li: [new Set(['li']), new Set([...LISTS, ...SCOPE])], + dt: [new Set(['dt', 'dd']), new Set(['dl', ...SCOPE])], + dd: [new Set(['dt', 'dd']), new Set(['dl', ...SCOPE])], + tr: [new Set(['tr']), new Set(['table', 'thead', 'tbody', 'tfoot'])], + td: [new Set(['td', 'th']), new Set(['tr', 'table'])], + th: [new Set(['td', 'th']), new Set(['tr', 'table'])], + body: [new Set(['head']), new Set()], +} +const P = new Set(['p']) +const TR = new Set(['tr']) +const TABLE = new Set(['table']) + +const DISPLAY_NONE = /(?:^|;)\s*display\s*:\s*none\s*(?:!important\s*)?(?:;|$)/i +const TAG_NAME = /[a-zA-Z][^ \t\n\f\r/>]*/y +const HTML_WS = /[ \t\n\f\r]+/ +// Sticky and possibly empty: `match` returns '' rather than null. +const WS = /[ \t\n\f\r]*/y +const ATTR_NAME = /[^ \t\n\f\r/>=]*/y +const UNQUOTED = /[^ \t\n\f\r>]*/y + +function match(re: RegExp, s: string, at: number): string { + re.lastIndex = at + return re.exec(s)?.[0] ?? '' +} + +interface Open { + name: string + /** 0 inline, 1 own line, 2 blank line around. */ + level: 0 | 1 | 2 + skip: boolean + /** Next number for an `

      `. */ + counter: number + /** Cells seen so far in a ``. */ + cells: number + href: string | null + /** Output length when an `` opened, to read its text back on close. */ + mark: number +} + +/** + * Accumulates output with lazy separators: a block only *requests* a break, + * and the break is written when (and if) the next real content arrives. That is + * what keeps empty `
      `s and leading/trailing blocks from leaving blank + * lines behind. + */ +class Writer { + out = '' + private trailingNewlines = 0 + private lineEmpty = true + private pendingBreak = 0 + private pendingSpace = false + /** List marker to print at the start of the next line. */ + marker = '' + /** Indentation for continuation lines inside a list item. */ + indent = '' + + requestBreak(level: number): void { + if (level > this.pendingBreak) this.pendingBreak = level + } + + space(): void { + this.pendingSpace = true + } + + word(s: string): void { + this.flush() + if (this.lineEmpty) this.startLine() + else if (this.pendingSpace) this.out += ' ' + this.put(s) + } + + /** Preformatted text: newlines are kept, nothing is collapsed. */ + verbatim(s: string): void { + this.flush() + s.split('\n').forEach((line, i) => { + if (i > 0) this.newline() + if (line === '') return + if (this.lineEmpty) this.startLine() + this.put(line) + }) + } + + newline(): void { + this.out += '\n' + this.trailingNewlines++ + this.lineEmpty = true + this.pendingSpace = false + } + + br(): void { + this.flush() + this.newline() + } + + tab(): void { + this.flush() + if (this.lineEmpty) this.startLine() + this.put('\t') + } + + private put(s: string): void { + this.out += s + this.trailingNewlines = 0 + this.lineEmpty = false + this.pendingSpace = false + } + + private flush(): void { + if (this.pendingBreak > 0 && this.out !== '') { + while (this.trailingNewlines < this.pendingBreak) this.newline() + } + this.pendingBreak = 0 + } + + private startLine(): void { + this.pendingSpace = false + if (this.marker !== '') { + this.out += this.marker + this.marker = '' + } else { + this.out += this.indent + } + } +} + +interface Tag { + name: string + attrs: Map + selfClosing: boolean + end: number +} + +/** Parse a start tag at `i` (which points at `<`). Null if it never closes. */ +function readStartTag(html: string, i: number): Tag | null { + TAG_NAME.lastIndex = i + 1 + const m = TAG_NAME.exec(html)! + const name = m[0].toLowerCase() + const attrs = new Map() + let j = i + 1 + m[0].length + let selfClosing = false + const n = html.length + + while (j < n) { + const c = html[j] + if (c === '>') return { name, attrs, selfClosing, end: j + 1 } + if (c === '/') { + selfClosing = html[j + 1] === '>' + j++ + continue + } + if (c === ' ' || c === '\t' || c === '\n' || c === '\f' || c === '\r') { + j++ + continue + } + selfClosing = false + + const attr = match(ATTR_NAME, html, j).toLowerCase() + j += attr.length + j += match(WS, html, j).length + + let value = '' + if (html[j] === '=') { + j++ + j += match(WS, html, j).length + const q = html[j] + if (q === '"' || q === "'") { + const close = html.indexOf(q, j + 1) + if (close === -1) return null + value = html.slice(j + 1, close) + j = close + 1 + } else { + value = match(UNQUOTED, html, j) + j += value.length + } + } + if (!attrs.has(attr)) attrs.set(attr, decodeEntities(value)) + } + return null +} + +function isHidden(attrs: Map, opts: ExtractOptions): boolean { + if (attrs.has('hidden')) return true + // role="img" makes the children presentational: this is what hides icon + // ligatures such as expand_less. + if (attrs.get('role')?.trim().split(HTML_WS)[0] === 'img') return true + const style = attrs.get('style') + if (style !== undefined && DISPLAY_NONE.test(style)) return true + if (opts.skipHiddenClasses) { + const cls = attrs.get('class') + if (cls !== undefined && cls.split(HTML_WS).some((c) => HIDDEN_CLASSES.has(c.toLowerCase()))) { + return true + } + } + return false +} + +function usefulHref(href: string | undefined): string | null { + if (href === undefined) return null + const h = href.trim() + if (h === '' || h.startsWith('#') || /^javascript:/i.test(h)) return null + return h +} + +/** + * Extract the readable text from an HTML fragment or document. + * + * A single-pass tokenizer with a light open-element stack -- enough to know + * which blocks, lists, cells and hidden subtrees we are inside, without + * building a tree. It never throws: malformed markup degrades the way a + * browser's would, or close to it. + */ +export function htmlToText(input: string, options: Partial = {}): string { + const opts = { ...DEFAULT_OPTIONS, ...options } + const html = input.replace(/\r\n?/g, '\n') + const n = html.length + const w = new Writer() + const stack: Open[] = [] + let skipDepth = 0 + let preDepth = 0 + let listDepth = 0 + let itemDepth = 0 + /** Set right after `
      `: its first newline is not content. */
      +  let preStart = false
      +
      +  const syncIndent = () => {
      +    w.indent = itemDepth > 0 ? '  '.repeat(listDepth) : ''
      +  }
      +
      +  const emitText = (raw: string) => {
      +    if (skipDepth > 0 || raw === '') return
      +    let text = decodeEntities(raw)
      +    if (preDepth > 0) {
      +      if (preStart && text.startsWith('\n')) text = text.slice(1)
      +      w.verbatim(text.replaceAll('\u00a0', ' '))
      +      return
      +    }
      +    text.split(HTML_WS).forEach((part, k) => {
      +      if (k > 0) w.space()
      +      if (part !== '') w.word(part.replaceAll('\u00a0', ' '))
      +    })
      +  }
      +
      +  const pop = () => {
      +    const el = stack.pop()!
      +    if (el.skip) {
      +      skipDepth--
      +      return
      +    }
      +    if (skipDepth > 0) return
      +    if (el.name === 'pre' || el.name === 'listing') preDepth--
      +    if (LISTS.has(el.name)) listDepth--
      +    if (el.name === 'li') itemDepth--
      +    syncIndent()
      +    if (el.href !== null && opts.linkUrls) {
      +      const text = w.out.slice(el.mark).trim()
      +      const bare = el.href.replace(/^mailto:/i, '')
      +      if (text !== el.href && text !== bare) {
      +        w.space()
      +        w.word(`<${el.href}>`)
      +      }
      +    }
      +    w.requestBreak(el.level)
      +  }
      +
      +  /** Index of the innermost open element in `names`, or -1 if a boundary comes first. */
      +  const findOpen = (names: ReadonlySet, boundary: ReadonlySet): number => {
      +    for (let k = stack.length - 1; k >= 0; k--) {
      +      const name = stack[k]!.name
      +      if (names.has(name)) return k
      +      if (boundary.has(name)) return -1
      +    }
      +    return -1
      +  }
      +
      +  const closeIfOpen = (names: ReadonlySet, boundary: ReadonlySet) => {
      +    const k = findOpen(names, boundary)
      +    if (k !== -1) while (stack.length > k) pop()
      +  }
      +
      +  const nearest = (names: ReadonlySet, boundary: ReadonlySet): Open | null =>
      +    stack[findOpen(names, boundary)] ?? null
      +
      +  const openTag = (tag: Tag) => {
      +    const { name, attrs } = tag
      +
      +    // Implicit closes, so unclosed 
    1. /

      / do not nest forever. + const implicit = Object.hasOwn(IMPLICIT, name) ? IMPLICIT[name] : undefined + if (implicit !== undefined) closeIfOpen(implicit[0], implicit[1]) + if (CLOSES_P.has(name)) closeIfOpen(P, SCOPE) + + const hidden = SKIP.has(name) || isHidden(attrs, opts) + + if (RAW_TEXT.has(name)) { + const close = new RegExp(`])`, 'gi') + close.lastIndex = tag.end + const m = close.exec(html) + const contentEnd = m === null ? n : m.index + if (!hidden) emitText(html.slice(tag.end, contentEnd)) + const gt = m === null ? -1 : html.indexOf('>', m.index) + return gt === -1 ? n : gt + 1 + } + + if (VOID.has(name)) { + if (skipDepth > 0 || hidden) return tag.end + if (name === 'br') w.br() + else if (name === 'hr') w.requestBreak(1) + return tag.end + } + + if (tag.selfClosing) return tag.end + + const el: Open = { name, level: 0, skip: false, counter: 1, cells: 0, href: null, mark: 0 } + stack.push(el) + if (hidden) { + el.skip = true + skipDepth++ + return tag.end + } + if (skipDepth > 0) return tag.end + + if (LISTS.has(name)) { + el.level = listDepth > 0 ? 1 : 2 + const start = Number.parseInt(attrs.get('start') ?? '', 10) + if (Number.isFinite(start)) el.counter = start + listDepth++ + } else if (PARAGRAPH.has(name)) { + el.level = 2 + } else if (LINE.has(name)) { + el.level = 1 + } + w.requestBreak(el.level) + + if (name === 'pre' || name === 'listing') { + preDepth++ + preStart = true + } else if (name === 'li') { + const list = nearest(LISTS, SCOPE) + const marker = list?.name === 'ol' ? `${list.counter++}. ` : '- ' + w.marker = ' '.repeat(Math.max(0, listDepth - 1)) + marker + itemDepth++ + } else if (name === 'td' || name === 'th') { + const row = nearest(TR, TABLE) + if (row !== null && row.cells++ > 0) w.tab() + } else if (name === 'a') { + el.href = usefulHref(attrs.get('href')) + el.mark = w.out.length + } + syncIndent() + return tag.end + } + + let i = 0 + while (i < n) { + const lt = html.indexOf('<', i) + if (lt === -1) { + emitText(html.slice(i)) + break + } + if (lt > i) { + emitText(html.slice(i, lt)) + preStart = false + } + i = lt + + const next = html[i + 1] + if (html.startsWith('` and `` are complete (empty) comments. + const end = html.startsWith('>', i + 4) + ? i + 4 + : html.startsWith('->', i + 4) + ? i + 5 + : html.indexOf('-->', i + 4) + i = end === -1 ? n : html[end] === '>' ? end + 1 : end + 3 + } else if (next === '!' || next === '?') { + const gt = html.indexOf('>', i) + i = gt === -1 ? n : gt + 1 + } else if (next === '/') { + TAG_NAME.lastIndex = i + 2 + const m = TAG_NAME.exec(html) + const gt = html.indexOf('>', i) + if (m !== null) { + const name = m[0].toLowerCase() + const k = stack.findLastIndex((el) => el.name === name) + if (k !== -1) while (stack.length > k) pop() + } + i = gt === -1 ? n : gt + 1 + } else if (next !== undefined && /[a-zA-Z]/.test(next)) { + const tag = readStartTag(html, i) + if (tag === null) break + i = openTag(tag) + if (tag.name !== 'pre' && tag.name !== 'listing') preStart = false + } else { + emitText('<') + i++ + } + } + + return w.out + .replace(/ +$/gm, '') + .replace(/^\n+|\n+$/g, '') +} diff --git a/src/tools/html-text/core/fixture-notification.html b/src/tools/html-text/core/fixture-notification.html new file mode 100644 index 0000000..100fe81 --- /dev/null +++ b/src/tools/html-text/core/fixture-notification.html @@ -0,0 +1 @@ +

      \ No newline at end of file