diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock new file mode 100644 index 0000000..0d3bf74 --- /dev/null +++ b/acceptance/crawl/contract.lock @@ -0,0 +1,5 @@ +1adcbc2fd87e3f508317f3ce24c3d48573aec657 +pending GET /healthz +pending POST /internal/v1/crawls/{crawl_id}/stop +held output_format: json +held discovery: pagination diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py index 1f1ba40..2203ce2 100644 --- a/acceptance/crawl/surface/test.py +++ b/acceptance/crawl/surface/test.py @@ -134,7 +134,7 @@ def crawls_since(t): ) subs = [ - ('"https://example.com/products/"', json.dumps(CRAWL_URL), "start URL → CRAWL_URL"), + ('"https://www.scrapingcourse.com/ecommerce/"', json.dumps(CRAWL_URL), "start URL → CRAWL_URL"), ("max_items=50,", "max_items=5,", "max_items 50 → 5"), ("max_pages=100,", "max_pages=10,", "max_pages 100 → 10 (at most 10)"), ( @@ -143,6 +143,17 @@ def crawls_since(t): "include → CRAWL_INCLUDE", ), ] +if API_BASE != "https://api.zenrows.com/v1": + # The quickstart leaves the base URL out, so it would reach production + # whatever CRAWL_API_BASE says. Point it where the rest of the run goes. + subs.append( + ( + 'ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"])', + 'ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"], ' + f"base_url={json.dumps(API_BASE)})", + "base URL → CRAWL_API_BASE", + ) + ) quick, applied = block or "", [] for a, b, why in subs: if a in quick: @@ -248,7 +259,8 @@ def do_GET(self): threading.Thread(target=srv.serve_forever, daemon=True).start() from zenrows import ZenRowsCrawlClient # noqa: E402 (imported once installed, as a user would) -from zenrows.crawl import CrawlAPIError # noqa: E402 +from zenrows.batch._open_enum import is_unknown # noqa: E402 +from zenrows.crawl import CrawlAPIError, CrawlErrorCode # noqa: E402 cap = ZenRowsCrawlClient( "capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0 @@ -839,6 +851,82 @@ def c21(): live=True, ) + +def c22(): + """discovery: sent as given, echoed back, and ["links"] when left out.""" + given = create(discovery=["links"], **BASE) + left_out = create(**{**BASE, "max_items": 1}) + for c in (given, left_out): + client.stop(c.crawl_id) + values = [[d.value for d in c.discovery] for c in (given, left_out)] + return ( + pass_if(values == [["links"], ["links"]]), + json.dumps({"discovery=['links']": values[0], "left out": values[1]}), + "", + ) + + +def c23(): + """content(download=True): the same page, asked for as a file to save.""" + + def first_fetched(crawl_id): + return next( + (r for r in client.get(crawl_id, limit=100).results if r.content_url), + None, + ) + + # The stop in check 20 can land before any of B's pages is fetched. Then read + # a one-item html crawl that runs to the end instead, as check 9 does. + b = state.get("B") + src = b.crawl_id if b else None + fetched = first_fetched(src) if src else None + if not fetched: + own = create(**{**BASE, "max_items": 1, "output_format": "html"}) + client.wait(own.crawl_id) + src = own.crawl_id + fetched = first_fetched(src) + if not fetched: + return FAIL, f"no result with a content_url on {src}", "" + plain = client.content(src, fetched) + as_file = client.content(src, fetched, download=True) + return ( + pass_if(plain == as_file and bool(plain)), + f"same {len(plain)} chars with and without download=True", + trim(plain, 400), + ) + + +def c24(): + """api_key_cap_reached parses as the named member, not as UNKNOWN.""" + code = CrawlErrorCode("api_key_cap_reached") + return ( + pass_if(code is CrawlErrorCode.API_KEY_CAP_REACHED and not is_unknown(code)), + f"{code!r} (name {code.name})", + "", + ) + + +run_check( + "22", + "Discovery", + 'create(discovery=["links"]) is echoed back, and a create without it reads ["links"]', + c22, + live=True, +) +run_check( + "23", + "Content as a file", + "content(download=True) returns the same page as content() does", + c23, + live=True, +) +run_check( + "24", + "Run error codes", + "api_key_cap_reached is a named CrawlErrorCode member, not an unknown value", + c24, +) + with open(f"{RESULTS}/naming.json", "w") as f: json.dump( { @@ -856,11 +944,13 @@ def c21(): "include_patterns", "exclude_patterns", "output_format", + "discovery", "idempotency_key", "cursor", "limit", "timeout", "content_id", + "download", ] }, }, diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py index 2f7eb56..d0e22e8 100644 --- a/src/zenrows/crawl/__init__.py +++ b/src/zenrows/crawl/__init__.py @@ -28,6 +28,7 @@ CrawlStatus, CrawlStop, CrawlWithResults, + DiscoveryStrategy, DownloadLine, OutputFormat, RunError, @@ -47,6 +48,7 @@ "CrawlStatus", "CrawlStop", "CrawlWithResults", + "DiscoveryStrategy", "DownloadLine", "OutputFormat", "RunError", diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index abac80b..7b911f9 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -172,6 +172,7 @@ def create( include_patterns: list[str] | None = None, exclude_patterns: list[str] | None = None, output_format: Literal["html", "markdown"] | None = None, + discovery: list[Literal["links"]] | None = None, idempotency_key: str | None = None, ) -> Crawl: """`POST /crawls` — start a crawl and return it, `running`. @@ -190,6 +191,9 @@ def create( - `output_format="html"` or `"markdown"`: also fetch every kept URL's page, as HTML or as Markdown, readable with `content` / `download`. Omit for URLs only. + - `discovery`: how the crawl finds the URLs it follows on each + page it walks. `["links"]` follows the links in the page's + HTML, and is what a crawl runs with when you leave it out. - `idempotency_key`: a retry with the same key and body answers with the crawl the first request created. @@ -206,6 +210,7 @@ def create( "include_patterns": include_patterns, "exclude_patterns": exclude_patterns, "output_format": output_format, + "discovery": discovery, } body.update({k: v for k, v in optional.items() if v is not None}) headers = {"Content-Type": "application/json"} @@ -300,7 +305,13 @@ def stop(self, crawl_id: str) -> CrawlStop: # ===== contents ===== - def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: + def content( + self, + crawl_id: str, + content_id: str | CrawlResult, + *, + download: bool = False, + ) -> str: """`GET /crawls/{crawl_id}/contents/{content_id}` — one kept URL's page, as text in the crawl's `output_format` (HTML or Markdown). @@ -310,6 +321,10 @@ def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: segment of the `content_url`). Only crawls created with an `output_format` have contents; anything else is 404 `content_not_found`. + + `download=True` asks for the page as a file to save: the server + sends `Content-Disposition: attachment` with a file name built + from the content id. The text returned is the same either way. """ if isinstance(content_id, CrawlResult): if not content_id.content_url: @@ -319,7 +334,8 @@ def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: ) content_id = content_id.content_url segment = content_id.rstrip("/").rsplit("/", 1)[-1] - return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}").text + params = {"download": "true"} if download else None + return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}", params=params).text def download(self, crawl_id: str) -> CrawlDownload: """`GET /crawls/{crawl_id}/download` — open the NDJSON file and diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py index 85050d3..35f8fda 100644 --- a/src/zenrows/crawl/models.py +++ b/src/zenrows/crawl/models.py @@ -12,7 +12,7 @@ from enum import Enum from typing import Any -from pydantic import BaseModel +from pydantic import BaseModel, Field from zenrows.batch._open_enum import open_enum_missing @@ -61,10 +61,23 @@ class OutputFormat(Enum): _missing_ = classmethod(open_enum_missing) +class DiscoveryStrategy(Enum): + """How the crawl finds the URLs it follows on each page it walks. + + `links` follows the links in the page's HTML. The server may run a + crawl with a strategy this version does not list, and it parses as + an `UNKNOWN` member keeping the raw value.""" + + LINKS = "links" + + _missing_ = classmethod(open_enum_missing) + + class CrawlErrorCode(Enum): """Why a crawl failed (`Crawl.error.code`).""" INSUFFICIENT_CREDITS = "insufficient_credits" + API_KEY_CAP_REACHED = "api_key_cap_reached" SEED_UNREACHABLE = "seed_unreachable" DOMAIN_NOT_ALLOWED = "domain_not_allowed" NO_ITEMS_FOUND = "no_items_found" @@ -104,6 +117,10 @@ class Crawl(BaseModel): created without one: it has no depth limit.""" max_items: int max_pages: int + discovery: list[DiscoveryStrategy] = Field(default_factory=lambda: [DiscoveryStrategy.LINKS]) + """The strategies the crawl finds URLs with. `["links"]` when the + create left it out, which is also what this reads when the server + does not send the field at all.""" coverage: Coverage created_at: datetime stop_reason: StopReason | None = None @@ -179,6 +196,7 @@ class DownloadLine(BaseModel): "CrawlStatus", "CrawlStop", "CrawlWithResults", + "DiscoveryStrategy", "DownloadLine", "OutputFormat", "RunError",