feat(scrape): add json output format to POST /web/scrape (#1213)

Stainless-Generated-From: dc8dff8070b08bc866dc0f1c079d18b22e91822d

Author: stlc-bot

src/context/dev/resources/web.py

@@ -392,6 +392,7 @@ def scrape(
         formats: web_scrape_params.Formats,
         url: str,
         image_params: web_scrape_params.ImageParams | Omit = omit,
+        json_params: web_scrape_params.JsonParams | Omit = omit,
         markdown_params: web_scrape_params.MarkdownParams | Omit = omit,
         max_age_ms: int | Omit = omit,
         parse_params: web_scrape_params.ParseParams | Omit = omit,
@@ -411,10 +412,12 @@ def scrape(
         visit.
 
         Each cache key includes only the settings that affect that output. HTML
-        is shared with Markdown and parsed fields. Cached outputs can come from
-        different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests
-        use the existing fast acquisition path. One credit per request, including cache
-        hits, or two with browser actions; PDF OCR adds one credit per recovered page on
+        is shared with Markdown, parsed fields, and JSON extraction. Cached outputs can
+        come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only
+        requests use the existing fast acquisition path. One credit per request,
+        including cache hits and missing pages, or two with browser actions; JSON
+        extraction adds four credits and runs an LLM over the page Markdown on every
+        request that has text to extract; PDF OCR adds one credit per recovered page on
         fresh extraction. Original response bytes and screenshots are limited to 20 MiB
         each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
 
@@ -425,6 +428,8 @@ def scrape(
 
           image_params: Image options. Requires formats.images: true.
 
+          json_params: Required when formats.json is true.
+
           markdown_params: Markdown options. Requires formats.markdown: true.
 
           max_age_ms: Maximum age of each cached output. Defaults to 1 day; 0 fetches fresh and
@@ -467,6 +472,7 @@ def scrape(
                     "formats": formats,
                     "url": url,
                     "image_params": image_params,
+                    "json_params": json_params,
                     "markdown_params": markdown_params,
                     "max_age_ms": max_age_ms,
                     "parse_params": parse_params,
@@ -1869,6 +1875,7 @@ async def scrape(
         formats: web_scrape_params.Formats,
         url: str,
         image_params: web_scrape_params.ImageParams | Omit = omit,
+        json_params: web_scrape_params.JsonParams | Omit = omit,
         markdown_params: web_scrape_params.MarkdownParams | Omit = omit,
         max_age_ms: int | Omit = omit,
         parse_params: web_scrape_params.ParseParams | Omit = omit,
@@ -1888,10 +1895,12 @@ async def scrape(
         visit.
 
         Each cache key includes only the settings that affect that output. HTML
-        is shared with Markdown and parsed fields. Cached outputs can come from
-        different visits within maxAgeMs; use 0 for a fresh capture. HTML-only requests
-        use the existing fast acquisition path. One credit per request, including cache
-        hits, or two with browser actions; PDF OCR adds one credit per recovered page on
+        is shared with Markdown, parsed fields, and JSON extraction. Cached outputs can
+        come from different visits within maxAgeMs; use 0 for a fresh capture. HTML-only
+        requests use the existing fast acquisition path. One credit per request,
+        including cache hits and missing pages, or two with browser actions; JSON
+        extraction adds four credits and runs an LLM over the page Markdown on every
+        request that has text to extract; PDF OCR adds one credit per recovered page on
         fresh extraction. Original response bytes and screenshots are limited to 20 MiB
         each, screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
 
@@ -1902,6 +1911,8 @@ async def scrape(
 
           image_params: Image options. Requires formats.images: true.
 
+          json_params: Required when formats.json is true.
+
           markdown_params: Markdown options. Requires formats.markdown: true.
 
           max_age_ms: Maximum age of each cached output. Defaults to 1 day; 0 fetches fresh and
@@ -1944,6 +1955,7 @@ async def scrape(
                     "formats": formats,
                     "url": url,
                     "image_params": image_params,
+                    "json_params": json_params,
                     "markdown_params": markdown_params,
                     "max_age_ms": max_age_ms,
                     "parse_params": parse_params,

src/context/dev/types/web_scrape_params.py

@@ -12,6 +12,7 @@
     "WebScrapeParams",
     "Formats",
     "ImageParams",
+    "JsonParams",
     "MarkdownParams",
     "ParseParams",
     "ParseParamsRules",
@@ -43,6 +44,9 @@ class WebScrapeParams(TypedDict, total=False):
     image_params: Annotated[ImageParams, PropertyInfo(alias="imageParams")]
     """Image options. Requires formats.images: true."""
 
+    json_params: Annotated[JsonParams, PropertyInfo(alias="jsonParams")]
+    """Required when formats.json is true."""
+
     markdown_params: Annotated[MarkdownParams, PropertyInfo(alias="markdownParams")]
     """Markdown options. Requires formats.markdown: true."""
 
@@ -100,6 +104,14 @@ class Formats(TypedDict, total=False):
     images: bool
     """Images found on the page."""
 
+    json: bool
+    """
+    Page data extracted by an LLM from the page Markdown into jsonParams.schema;
+    values carried only in attributes or CSS classes need formats.parse instead.
+    Adds four credits when the page has text to extract; when shared content filters
+    leave no text the result is an empty object and only the base price applies.
+    """
+
     markdown: bool
     """Page content as Markdown."""
 
@@ -124,6 +136,26 @@ class ImageParams(TypedDict, total=False):
     """
 
 
+class JsonParams(TypedDict, total=False):
+    """Required when formats.json is true."""
+
+    schema: Required[Dict[str, object]]
+    """JSON Schema for the returned object.
+
+    Must describe a top-level object; at most 50 KB serialized. Optional fields the
+    page does not state are omitted, or null when their type allows null, while
+    required non-nullable fields always receive a best-effort value, so prefer
+    nullable or optional fields for data a page may omit. Zod users can pass the
+    output of z.toJSONSchema().
+    """
+
+    instructions: str
+    """
+    Optional guidance on which facts to prioritize or how to interpret schema
+    fields.
+    """
+
+
 class MarkdownParams(TypedDict, total=False):
     """Markdown options. Requires formats.markdown: true."""
 

src/context/dev/types/web_scrape_response.py

@@ -15,6 +15,7 @@
     "HTML",
     "Images",
     "ImagesData",
+    "Json",
     "Markdown",
     "Metadata",
     "MetadataAlternate",
@@ -100,6 +101,17 @@ class Images(BaseModel):
     requested: bool
 
 
+class Json(BaseModel):
+    """Page data extracted into jsonParams.schema, after shared content filters.
+
+    Values are grounded in the page; optional fields the page does not state are omitted, or null when their type allows null. An empty object when the filters leave no text.
+    """
+
+    data: Optional[Dict[str, object]] = None
+
+    requested: bool
+
+
 class Markdown(BaseModel):
     """Markdown after content filters."""
 
@@ -237,6 +249,14 @@ class WebScrapeResponse(BaseModel):
     images: Images
     """Images after content filters. Empty when none are found."""
 
+    json_: Json = FieldInfo(alias="json")
+    """Page data extracted into jsonParams.schema, after shared content filters.
+
+    Values are grounded in the page; optional fields the page does not state are
+    omitted, or null when their type allows null. An empty object when the filters
+    leave no text.
+    """
+
     markdown: Markdown
     """Markdown after content filters."""
 

tests/api_resources/test_web.py

@@ -244,6 +244,7 @@ def test_method_scrape_with_all_params(self, client: ContextDev) -> None:
                 "bytes": True,
                 "html": True,
                 "images": True,
+                "json": True,
                 "markdown": True,
                 "parse": True,
                 "screenshot": True,
@@ -253,6 +254,15 @@ def test_method_scrape_with_all_params(self, client: ContextDev) -> None:
                 "dedupe": "none",
                 "enrich": ["dimensions"],
             },
+            json_params={
+                "schema": {
+                    "type": "bar",
+                    "properties": "bar",
+                    "required": "bar",
+                    "additionalProperties": "bar",
+                },
+                "instructions": "instructions",
+            },
             markdown_params={
                 "include_images": True,
                 "include_links": True,
@@ -764,6 +774,7 @@ async def test_method_scrape_with_all_params(self, async_client: AsyncContextDev
                 "bytes": True,
                 "html": True,
                 "images": True,
+                "json": True,
                 "markdown": True,
                 "parse": True,
                 "screenshot": True,
@@ -773,6 +784,15 @@ async def test_method_scrape_with_all_params(self, async_client: AsyncContextDev
                 "dedupe": "none",
                 "enrich": ["dimensions"],
             },
+            json_params={
+                "schema": {
+                    "type": "bar",
+                    "properties": "bar",
+                    "required": "bar",
+                    "additionalProperties": "bar",
+                },
+                "instructions": "instructions",
+            },
             markdown_params={
                 "include_images": True,
                 "include_links": True,