diff --git a/.changeset/ingest-on-workers.md b/.changeset/ingest-on-workers.md new file mode 100644 index 0000000..0ccfef9 --- /dev/null +++ b/.changeset/ingest-on-workers.md @@ -0,0 +1,5 @@ +--- +'@chatcops/ingest': patch +--- + +Ingest runs on Cloudflare Workers. Pages were converted to Markdown by turndown's browser build there, which needs a `document` a Worker doesn't have, so every page failed with "document is not defined". turndown now gets a page parsed by domino, the parser its Node build already uses, so the Markdown is the same everywhere. diff --git a/packages/ingest/package.json b/packages/ingest/package.json index 91cfa10..5757f26 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -47,6 +47,7 @@ "node": ">=22.13" }, "dependencies": { + "@mixmark-io/domino": "^2.2.0", "@mozilla/readability": "^0.6.0", "linkedom": "^0.18.12", "turndown": "^7.2.0", diff --git a/packages/ingest/src/extract.ts b/packages/ingest/src/extract.ts index 0f2aa37..6b877ce 100644 --- a/packages/ingest/src/extract.ts +++ b/packages/ingest/src/extract.ts @@ -1,3 +1,4 @@ +import domino from '@mixmark-io/domino'; import { Readability, isProbablyReaderable } from '@mozilla/readability'; import { parseHTML } from 'linkedom'; import TurndownService from 'turndown'; @@ -36,6 +37,17 @@ turndown.use(gfm); turndown.addRule('no-images', { filter: 'img', replacement: () => '' }); turndown.addRule('link-text', { filter: 'a', replacement: (content) => content }); +/** + * Markdown from an HTML fragment. turndown gets an element domino parsed, + * not a string: given a string, its browser build (the one Workers bundles + * pick) parses with the page's `document`, and a Worker has none. domino is + * the parser turndown's Node build uses anyway, so the output is the same. + */ +function toMarkdown(html: string): string { + const document = domino.createDocument(`${html}`); + return turndown.turndown(document.body as unknown as HTMLElement); +} + /** * The readable content of an HTML page as Markdown. Hidden elements and * comments go first. Then `selector`, when given, picks the content; @@ -77,7 +89,7 @@ export function extractHtml(html: string, url: string, selector?: string): Extra ...(title ? { title } : {}), ...(lang ? { lang } : {}), ...(description ? { description } : {}), - markdown: tidy(turndown.turndown(contentHtml)), + markdown: tidy(toMarkdown(contentHtml)), }; } diff --git a/packages/ingest/src/mixmark-domino.d.ts b/packages/ingest/src/mixmark-domino.d.ts new file mode 100644 index 0000000..f32b803 --- /dev/null +++ b/packages/ingest/src/mixmark-domino.d.ts @@ -0,0 +1,5 @@ +// The package's own types declare the module under its old name, 'domino'. +declare module '@mixmark-io/domino' { + const domino: { createDocument(html?: string, force?: boolean): Document }; + export default domino; +} diff --git a/packages/ingest/tests/extract-workers.test.ts b/packages/ingest/tests/extract-workers.test.ts new file mode 100644 index 0000000..15a8fc9 --- /dev/null +++ b/packages/ingest/tests/extract-workers.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it, vi } from 'vitest'; +import { extractHtml } from '../src/index.js'; +import { PAGES } from './site.js'; + +// Workers bundles pick turndown's browser build, which parses a string with +// the page's `document`. A Worker has none, and neither does this test. +// @ts-expect-error The browser build ships without its own types. +vi.mock('turndown', async () => import('turndown/lib/turndown.browser.es.js')); + +describe('extractHtml with the browser build of turndown', () => { + it('converts a page with no global document, tables included', () => { + expect(typeof (globalThis as { document?: unknown }).document).toBe('undefined'); + const doc = extractHtml(PAGES['/pricing'], 'https://acme.test/pricing'); + expect(doc.markdown).toContain('The Team plan costs $20 per user per month'); + expect(doc.markdown).toContain('## Enterprise'); + expect(doc.markdown).toMatch(/\| Plan \| Price \|/); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index db7db67..ca624b7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -94,6 +94,9 @@ importers: packages/ingest: dependencies: + '@mixmark-io/domino': + specifier: ^2.2.0 + version: 2.2.0 '@mozilla/readability': specifier: ^0.6.0 version: 0.6.0