fix(ingest): run on Cloudflare Workers

.changeset/ingest-on-workers.md

@@ -0,0 +1,5 @@
+---
+'@chatcops/ingest': patch
+---
+
+Ingest runs on Cloudflare Workers. Pages were converted to Markdown by turndown's browser build there, which needs a `document` a Worker doesn't have, so every page failed with "document is not defined". turndown now gets a page parsed by domino, the parser its Node build already uses, so the Markdown is the same everywhere.

packages/ingest/package.json

@@ -47,6 +47,7 @@
     "node": ">=22.13"
   },
   "dependencies": {
+    "@mixmark-io/domino": "^2.2.0",
     "@mozilla/readability": "^0.6.0",
     "linkedom": "^0.18.12",
     "turndown": "^7.2.0",

packages/ingest/src/extract.ts

@@ -1,3 +1,4 @@
+import domino from '@mixmark-io/domino';
 import { Readability, isProbablyReaderable } from '@mozilla/readability';
 import { parseHTML } from 'linkedom';
 import TurndownService from 'turndown';
@@ -36,6 +37,17 @@ turndown.use(gfm);
 turndown.addRule('no-images', { filter: 'img', replacement: () => '' });
 turndown.addRule('link-text', { filter: 'a', replacement: (content) => content });
 
+/**
+ * Markdown from an HTML fragment. turndown gets an element domino parsed,
+ * not a string: given a string, its browser build (the one Workers bundles
+ * pick) parses with the page's `document`, and a Worker has none. domino is
+ * the parser turndown's Node build uses anyway, so the output is the same.
+ */
+function toMarkdown(html: string): string {
+  const document = domino.createDocument(`<!doctype html><html><body>${html}</body></html>`);
+  return turndown.turndown(document.body as unknown as HTMLElement);
+}
+
 /**
  * The readable content of an HTML page as Markdown. Hidden elements and
  * comments go first. Then `selector`, when given, picks the content;
@@ -77,7 +89,7 @@ export function extractHtml(html: string, url: string, selector?: string): Extra
     ...(title ? { title } : {}),
     ...(lang ? { lang } : {}),
     ...(description ? { description } : {}),
-    markdown: tidy(turndown.turndown(contentHtml)),
+    markdown: tidy(toMarkdown(contentHtml)),
   };
 }
 

packages/ingest/src/mixmark-domino.d.ts

@@ -0,0 +1,5 @@
+// The package's own types declare the module under its old name, 'domino'.
+declare module '@mixmark-io/domino' {
+  const domino: { createDocument(html?: string, force?: boolean): Document };
+  export default domino;
+}

packages/ingest/tests/extract-workers.test.ts

@@ -0,0 +1,18 @@
+import { describe, expect, it, vi } from 'vitest';
+import { extractHtml } from '../src/index.js';
+import { PAGES } from './site.js';
+
+// Workers bundles pick turndown's browser build, which parses a string with
+// the page's `document`. A Worker has none, and neither does this test.
+// @ts-expect-error The browser build ships without its own types.
+vi.mock('turndown', async () => import('turndown/lib/turndown.browser.es.js'));
+
+describe('extractHtml with the browser build of turndown', () => {
+  it('converts a page with no global document, tables included', () => {
+    expect(typeof (globalThis as { document?: unknown }).document).toBe('undefined');
+    const doc = extractHtml(PAGES['/pricing'], 'https://acme.test/pricing');
+    expect(doc.markdown).toContain('The Team plan costs $20 per user per month');
+    expect(doc.markdown).toContain('## Enterprise');
+    expect(doc.markdown).toMatch(/\| Plan \| Price \|/);
+  });
+});