Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -505,8 +505,9 @@ link hops from the start URL; without it the crawl has no depth limit.
It stays on the start URL's registrable domain;
subdomains count. It runs in the background: `create` returns at once with
`status="running"`, and `wait` blocks until it ends. Without
`output_format` a crawl returns URLs only; `output_format="html"` also
returns each kept URL's page.
`output_format` a crawl returns URLs only; `output_format="html"` or
`output_format="markdown"` also returns each kept URL's page, as HTML or
as Markdown.

```python
import os
Expand Down
4 changes: 2 additions & 2 deletions src/zenrows/crawl/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
- `models` — hand-written pydantic v2 response models with open enums.
- `errors.CrawlAPIError` — RFC 9457 problem+json mapping.

`create(output_format="html")` also returns each kept URL's page;
without it a crawl returns URLs only.
`create(output_format="html")` or `create(output_format="markdown")` also
returns each kept URL's page; without it a crawl returns URLs only.
"""

from zenrows.crawl.client import (
Expand Down
10 changes: 6 additions & 4 deletions src/zenrows/crawl/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ def create(
max_pages: int | None = None,
include_patterns: list[str] | None = None,
exclude_patterns: list[str] | None = None,
output_format: Literal["html"] | None = None,
output_format: Literal["html", "markdown"] | None = None,
idempotency_key: str | None = None,
) -> Crawl:
"""`POST /crawls` — start a crawl and return it, `running`.
Expand All @@ -187,8 +187,9 @@ def create(
against each normalized URL. A URL is kept when it matches
an include pattern (if any) and no exclude pattern. A crawl
stays on the start URL's registrable domain; subdomains count.
- `output_format="html"`: also fetch every kept URL's page,
readable with `content` / `download`. Omit for URLs only.
- `output_format="html"` or `"markdown"`: also fetch every kept
URL's page, as HTML or as Markdown, readable with `content` /
`download`. Omit for URLs only.
- `idempotency_key`: a retry with the same key and body answers
with the crawl the first request created.

Expand Down Expand Up @@ -301,7 +302,8 @@ def stop(self, crawl_id: str) -> CrawlStop:

def content(self, crawl_id: str, content_id: str | CrawlResult) -> str:
"""`GET /crawls/{crawl_id}/contents/{content_id}` — one kept
URL's page, as HTML text.
URL's page, as text in the crawl's `output_format` (HTML or
Markdown).

`content_id` is a `CrawlResult` whose `content_status` is
`fetched`, its `content_url`, or its content id (the last
Expand Down
1 change: 1 addition & 0 deletions src/zenrows/crawl/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ class OutputFormat(Enum):
"""The format the crawl returns each kept URL's page in."""

HTML = "html"
MARKDOWN = "markdown"

_missing_ = classmethod(open_enum_missing)

Expand Down
24 changes: 24 additions & 0 deletions tests/test_crawl_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,21 @@ def test_create_sends_every_option_and_idempotency_key(client: ZenRowsCrawlClien
assert "discovery" not in body


@respx.mock
def test_create_sends_markdown_output_format(client: ZenRowsCrawlClient):
route = respx.post(f"{BASE_URL}/crawls").mock(
return_value=Response(202, json=crawl_body(output_format="markdown"))
)

crawl = client.create("https://example.com/", output_format="markdown")

assert crawl.output_format is OutputFormat.MARKDOWN
assert json.loads(route.calls.last.request.content) == {
"url": "https://example.com/",
"output_format": "markdown",
}


# ----- get / list / stop -----


Expand Down Expand Up @@ -416,6 +431,15 @@ def test_content_accepts_result_url_or_id(client: ZenRowsCrawlClient):
assert route.call_count == 3


@respx.mock
def test_content_returns_markdown_text(client: ZenRowsCrawlClient):
respx.get(f"{BASE_URL}/crawls/c_1/contents/ct_a").mock(
return_value=Response(200, headers={"Content-Type": "text/markdown"}, text="# Product")
)

assert client.content("c_1", "ct_a") == "# Product"


def test_content_without_content_url_raises(client: ZenRowsCrawlClient):
with pytest.raises(ValueError, match="content_url"):
client.content("c_1", CrawlResult(url="https://a", content_status="pending")) # type: ignore[arg-type]
Expand Down
Loading