From 2daacdfccffd40d9620533ff66dbc6bc6fc93f11 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Wed, 14 Oct 2026 11:13:43 +0200 Subject: [PATCH] feat(crawl): accept markdown as an output format The Crawl API now returns each kept page as Markdown when a crawl is created with output_format="markdown". create accepts it, OutputFormat names it, and content returns the Markdown text. --- README.md | 5 +++-- src/zenrows/crawl/__init__.py | 4 ++-- src/zenrows/crawl/client.py | 10 ++++++---- src/zenrows/crawl/models.py | 1 + tests/test_crawl_client.py | 24 ++++++++++++++++++++++++ 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 94404fe..6fd860b 100644 --- a/README.md +++ b/README.md @@ -505,8 +505,9 @@ link hops from the start URL; without it the crawl has no depth limit. It stays on the start URL's registrable domain; subdomains count. It runs in the background: `create` returns at once with `status="running"`, and `wait` blocks until it ends. Without -`output_format` a crawl returns URLs only; `output_format="html"` also -returns each kept URL's page. +`output_format` a crawl returns URLs only; `output_format="html"` or +`output_format="markdown"` also returns each kept URL's page, as HTML or +as Markdown. ```python import os diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py index 696a9d7..2f7eb56 100644 --- a/src/zenrows/crawl/__init__.py +++ b/src/zenrows/crawl/__init__.py @@ -9,8 +9,8 @@ - `models` — hand-written pydantic v2 response models with open enums. - `errors.CrawlAPIError` — RFC 9457 problem+json mapping. -`create(output_format="html")` also returns each kept URL's page; -without it a crawl returns URLs only. +`create(output_format="html")` or `create(output_format="markdown")` also +returns each kept URL's page; without it a crawl returns URLs only. """ from zenrows.crawl.client import ( diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 20bf213..abac80b 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -171,7 +171,7 @@ def create( max_pages: int | None = None, include_patterns: list[str] | None = None, exclude_patterns: list[str] | None = None, - output_format: Literal["html"] | None = None, + output_format: Literal["html", "markdown"] | None = None, idempotency_key: str | None = None, ) -> Crawl: """`POST /crawls` — start a crawl and return it, `running`. @@ -187,8 +187,9 @@ def create( against each normalized URL. A URL is kept when it matches an include pattern (if any) and no exclude pattern. A crawl stays on the start URL's registrable domain; subdomains count. - - `output_format="html"`: also fetch every kept URL's page, - readable with `content` / `download`. Omit for URLs only. + - `output_format="html"` or `"markdown"`: also fetch every kept + URL's page, as HTML or as Markdown, readable with `content` / + `download`. Omit for URLs only. - `idempotency_key`: a retry with the same key and body answers with the crawl the first request created. @@ -301,7 +302,8 @@ def stop(self, crawl_id: str) -> CrawlStop: def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: """`GET /crawls/{crawl_id}/contents/{content_id}` — one kept - URL's page, as HTML text. + URL's page, as text in the crawl's `output_format` (HTML or + Markdown). `content_id` is a `CrawlResult` whose `content_status` is `fetched`, its `content_url`, or its content id (the last diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index e1ab707..85050d3 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -56,6 +56,7 @@ class OutputFormat(Enum): """The format the crawl returns each kept URL's page in.""" HTML = "html" + MARKDOWN = "markdown" _missing_ = classmethod(open_enum_missing) diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index bed4cc9..962022d 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -158,6 +158,21 @@ def test_create_sends_every_option_and_idempotency_key(client: ZenRowsCrawlClien assert "discovery" not in body +@respx.mock +def test_create_sends_markdown_output_format(client: ZenRowsCrawlClient): + route = respx.post(f"{BASE_URL}/crawls").mock( + return_value=Response(202, json=crawl_body(output_format="markdown")) + ) + + crawl = client.create("https://example.com/", output_format="markdown") + + assert crawl.output_format is OutputFormat.MARKDOWN + assert json.loads(route.calls.last.request.content) == { + "url": "https://example.com/", + "output_format": "markdown", + } + + # ----- get / list / stop ----- @@ -416,6 +431,15 @@ def test_content_accepts_result_url_or_id(client: ZenRowsCrawlClient): assert route.call_count == 3 +@respx.mock +def test_content_returns_markdown_text(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/c_1/contents/ct_a").mock( + return_value=Response(200, headers={"Content-Type": "text/markdown"}, text="# Product") + ) + + assert client.content("c_1", "ct_a") == "# Product" + + def test_content_without_content_url_raises(client: ZenRowsCrawlClient): with pytest.raises(ValueError, match="content_url"): client.content("c_1", CrawlResult(url="https://a", content_status="pending")) # type: ignore[arg-type]