feat(crawl): read the discovery strategies a crawl ran with, and save a page as a file

acceptance/crawl/contract.lock

@@ -0,0 +1,3 @@
+1adcbc2fd87e3f508317f3ce24c3d48573aec657
+pending GET /healthz
+pending POST /internal/v1/crawls/{crawl_id}/stop

acceptance/crawl/surface/test.py

@@ -248,7 +248,8 @@ def do_GET(self):
 threading.Thread(target=srv.serve_forever, daemon=True).start()
 
 from zenrows import ZenRowsCrawlClient  # noqa: E402  (imported once installed, as a user would)
-from zenrows.crawl import CrawlAPIError  # noqa: E402
+from zenrows.batch._open_enum import is_unknown  # noqa: E402
+from zenrows.crawl import CrawlAPIError, CrawlErrorCode  # noqa: E402
 
 cap = ZenRowsCrawlClient(
     "capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0
@@ -839,6 +840,82 @@ def c21():
     live=True,
 )
 
+
+def c22():
+    """discovery: sent as given, echoed back, and ["links"] when left out."""
+    given = create(discovery=["links"], **BASE)
+    left_out = create(**{**BASE, "max_items": 1})
+    for c in (given, left_out):
+        client.stop(c.crawl_id)
+    values = [[d.value for d in c.discovery] for c in (given, left_out)]
+    return (
+        pass_if(values == [["links"], ["links"]]),
+        json.dumps({"discovery=['links']": values[0], "left out": values[1]}),
+        "",
+    )
+
+
+def c23():
+    """content(download=True): the same page, asked for as a file to save."""
+
+    def first_fetched(crawl_id):
+        return next(
+            (r for r in client.get(crawl_id, limit=100).results if r.content_url),
+            None,
+        )
+
+    # The stop in check 20 can land before any of B's pages is fetched. Then read
+    # a one-item html crawl that runs to the end instead, as check 9 does.
+    b = state.get("B")
+    src = b.crawl_id if b else None
+    fetched = first_fetched(src) if src else None
+    if not fetched:
+        own = create(**{**BASE, "max_items": 1, "output_format": "html"})
+        client.wait(own.crawl_id)
+        src = own.crawl_id
+        fetched = first_fetched(src)
+    if not fetched:
+        return FAIL, f"no result with a content_url on {src}", ""
+    plain = client.content(src, fetched)
+    as_file = client.content(src, fetched, download=True)
+    return (
+        pass_if(plain == as_file and bool(plain)),
+        f"same {len(plain)} chars with and without download=True",
+        trim(plain, 400),
+    )
+
+
+def c24():
+    """api_key_cap_reached parses as the named member, not as UNKNOWN."""
+    code = CrawlErrorCode("api_key_cap_reached")
+    return (
+        pass_if(code is CrawlErrorCode.API_KEY_CAP_REACHED and not is_unknown(code)),
+        f"{code!r} (name {code.name})",
+        "",
+    )
+
+
+run_check(
+    "22",
+    "Discovery",
+    'create(discovery=["links"]) is echoed back, and a create without it reads ["links"]',
+    c22,
+    live=True,
+)
+run_check(
+    "23",
+    "Content as a file",
+    "content(download=True) returns the same page as content() does",
+    c23,
+    live=True,
+)
+run_check(
+    "24",
+    "Run error codes",
+    "api_key_cap_reached is a named CrawlErrorCode member, not an unknown value",
+    c24,
+)
+
 with open(f"{RESULTS}/naming.json", "w") as f:
     json.dump(
         {
@@ -856,11 +933,13 @@ def c21():
                     "include_patterns",
                     "exclude_patterns",
                     "output_format",
+                    "discovery",
                     "idempotency_key",
                     "cursor",
                     "limit",
                     "timeout",
                     "content_id",
+                    "download",
                 ]
             },
         },

src/zenrows/crawl/__init__.py

@@ -28,6 +28,7 @@
     CrawlStatus,
     CrawlStop,
     CrawlWithResults,
+    DiscoveryStrategy,
     DownloadLine,
     OutputFormat,
     RunError,
@@ -47,6 +48,7 @@
     "CrawlStatus",
     "CrawlStop",
     "CrawlWithResults",
+    "DiscoveryStrategy",
     "DownloadLine",
     "OutputFormat",
     "RunError",

src/zenrows/crawl/client.py

@@ -172,6 +172,7 @@ def create(
         include_patterns: list[str] | None = None,
         exclude_patterns: list[str] | None = None,
         output_format: Literal["html", "markdown"] | None = None,
+        discovery: list[Literal["links"]] | None = None,
         idempotency_key: str | None = None,
     ) -> Crawl:
         """`POST /crawls` — start a crawl and return it, `running`.
@@ -190,6 +191,9 @@ def create(
         - `output_format="html"` or `"markdown"`: also fetch every kept
           URL's page, as HTML or as Markdown, readable with `content` /
           `download`. Omit for URLs only.
+        - `discovery`: how the crawl finds the URLs it follows on each
+          page it walks. `["links"]` follows the links in the page's
+          HTML, and is what a crawl runs with when you leave it out.
         - `idempotency_key`: a retry with the same key and body answers
           with the crawl the first request created.
 
@@ -206,6 +210,7 @@ def create(
             "include_patterns": include_patterns,
             "exclude_patterns": exclude_patterns,
             "output_format": output_format,
+            "discovery": discovery,
         }
         body.update({k: v for k, v in optional.items() if v is not None})
         headers = {"Content-Type": "application/json"}
@@ -300,7 +305,13 @@ def stop(self, crawl_id: str) -> CrawlStop:
 
     # ===== contents =====
 
-    def content(self, crawl_id: str, content_id: str | CrawlResult) -> str:
+    def content(
+        self,
+        crawl_id: str,
+        content_id: str | CrawlResult,
+        *,
+        download: bool = False,
+    ) -> str:
         """`GET /crawls/{crawl_id}/contents/{content_id}` — one kept
         URL's page, as text in the crawl's `output_format` (HTML or
         Markdown).
@@ -310,6 +321,10 @@ def content(self, crawl_id: str, content_id: str | CrawlResult) -> str:
         segment of the `content_url`). Only crawls created with an
         `output_format` have contents; anything else is 404
         `content_not_found`.
+
+        `download=True` asks for the page as a file to save: the server
+        sends `Content-Disposition: attachment` with a file name built
+        from the content id. The text returned is the same either way.
         """
         if isinstance(content_id, CrawlResult):
             if not content_id.content_url:
@@ -319,7 +334,8 @@ def content(self, crawl_id: str, content_id: str | CrawlResult) -> str:
                 )
             content_id = content_id.content_url
         segment = content_id.rstrip("/").rsplit("/", 1)[-1]
-        return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}").text
+        params = {"download": "true"} if download else None
+        return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}", params=params).text
 
     def download(self, crawl_id: str) -> CrawlDownload:
         """`GET /crawls/{crawl_id}/download` — open the NDJSON file and

src/zenrows/crawl/models.py

@@ -12,7 +12,7 @@
 from enum import Enum
 from typing import Any
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from zenrows.batch._open_enum import open_enum_missing
 
@@ -61,10 +61,23 @@ class OutputFormat(Enum):
     _missing_ = classmethod(open_enum_missing)
 
 
+class DiscoveryStrategy(Enum):
+    """How the crawl finds the URLs it follows on each page it walks.
+
+    `links` follows the links in the page's HTML. The server may run a
+    crawl with a strategy this version does not list, and it parses as
+    an `UNKNOWN` member keeping the raw value."""
+
+    LINKS = "links"
+
+    _missing_ = classmethod(open_enum_missing)
+
+
 class CrawlErrorCode(Enum):
     """Why a crawl failed (`Crawl.error.code`)."""
 
     INSUFFICIENT_CREDITS = "insufficient_credits"
+    API_KEY_CAP_REACHED = "api_key_cap_reached"
     SEED_UNREACHABLE = "seed_unreachable"
     DOMAIN_NOT_ALLOWED = "domain_not_allowed"
     NO_ITEMS_FOUND = "no_items_found"
@@ -104,6 +117,10 @@ class Crawl(BaseModel):
     created without one: it has no depth limit."""
     max_items: int
     max_pages: int
+    discovery: list[DiscoveryStrategy] = Field(default_factory=lambda: [DiscoveryStrategy.LINKS])
+    """The strategies the crawl finds URLs with. `["links"]` when the
+    create left it out, which is also what this reads when the server
+    does not send the field at all."""
     coverage: Coverage
     created_at: datetime
     stop_reason: StopReason | None = None
@@ -179,6 +196,7 @@ class DownloadLine(BaseModel):
     "CrawlStatus",
     "CrawlStop",
     "CrawlWithResults",
+    "DiscoveryStrategy",
     "DownloadLine",
     "OutputFormat",
     "RunError",