feat(scrape): add CSS extraction rules to HTML scraping (#1146)

Stainless-Generated-From: 9c6de15dc33c67452e013870363c056f393e3cfe

Author: stlc-bot

src/context/dev/resources/web.py

@@ -1977,6 +1977,7 @@ def web_scrape_html(
         ]
         | Omit = omit,
         exclude_selectors: Optional[SequenceNotStr[str]] | Omit = omit,
+        extract_rules: Dict[str, web_web_scrape_html_params.ExtractRules] | Omit = omit,
         headers: Dict[str, str] | Omit = omit,
         include_frames: bool | Omit = omit,
         include_selectors: Optional[SequenceNotStr[str]] | Omit = omit,
@@ -1995,14 +1996,18 @@ def web_scrape_html(
         extra_body: Body | None = None,
         timeout: float | httpx.Timeout | None | NotGiven = not_given,
     ) -> WebWebScrapeHTMLResponse:
-        """Scrapes the given URL and returns the raw HTML content of the page.
-
-        The base
-        request costs 1 credit; requests with browser actions cost 2 credits. A request
-        that hits its timeoutOpts.milliseconds deadline fails with 408 and is not
-        billed, unless timeoutOpts.behavior=return-partial is set — then the page as
-        rendered so far is returned with `finalDOMState: "still-loading"` and billed at
-        the base cost of 1 credit.
+        """Scrapes the given URL and returns the HTML content of the page.
+
+        Optional
+        extractRules return deterministic structured data in extracted using CSS
+        selectors, attributes, lists, and nested rules, without an LLM or additional
+        credits. Rules run on the returned HTML after selector and main-content
+        filtering. Send extractRules as a JSON-encoded query parameter. The base request
+        costs 1 credit; requests with browser actions cost 2 credits. A request that
+        hits its timeoutOpts.milliseconds deadline fails with 408 and is not billed,
+        unless timeoutOpts.behavior=return-partial is set — then the page as rendered so
+        far is returned with `finalDOMState: "still-loading"` and billed at the base
+        cost of 1 credit.
 
         Args:
           url: Full URL to scrape (must include http:// or https:// protocol)
@@ -2018,6 +2023,14 @@ def web_scrape_html(
               Exclusion takes precedence: an element matching both is removed. Examples:
               "nav", "footer", ".ad-banner", "[aria-hidden=true]".
 
+          extract_rules: Optional CSS extraction rules applied to the returned HTML after selector and
+              main-content filtering. Use selector strings ("h1", "a@href") or objects with
+              selector, type (item or list), and output (text, html, @attribute, or nested
+              rules). Text whitespace is normalized; html includes the matched element;
+              attributes are returned as written. Missing items are null and missing lists are
+              empty. CSS only; XPath is not supported. Maximum: 100 fields across 5 levels.
+              Send a JSON-encoded string in the extractRules query parameter.
+
           headers: Optional outbound HTTP headers forwarded only to the target URL, sent as
               deep-object query params such as headers[X-Custom]=value. When provided, caching
               is bypassed: the result is neither read from nor written to cache.
@@ -2081,6 +2094,7 @@ def web_scrape_html(
                         "actions": actions,
                         "country": country,
                         "exclude_selectors": exclude_selectors,
+                        "extract_rules": extract_rules,
                         "headers": headers,
                         "include_frames": include_frames,
                         "include_selectors": include_selectors,
@@ -4594,6 +4608,7 @@ async def web_scrape_html(
         ]
         | Omit = omit,
         exclude_selectors: Optional[SequenceNotStr[str]] | Omit = omit,
+        extract_rules: Dict[str, web_web_scrape_html_params.ExtractRules] | Omit = omit,
         headers: Dict[str, str] | Omit = omit,
         include_frames: bool | Omit = omit,
         include_selectors: Optional[SequenceNotStr[str]] | Omit = omit,
@@ -4612,14 +4627,18 @@ async def web_scrape_html(
         extra_body: Body | None = None,
         timeout: float | httpx.Timeout | None | NotGiven = not_given,
     ) -> WebWebScrapeHTMLResponse:
-        """Scrapes the given URL and returns the raw HTML content of the page.
-
-        The base
-        request costs 1 credit; requests with browser actions cost 2 credits. A request
-        that hits its timeoutOpts.milliseconds deadline fails with 408 and is not
-        billed, unless timeoutOpts.behavior=return-partial is set — then the page as
-        rendered so far is returned with `finalDOMState: "still-loading"` and billed at
-        the base cost of 1 credit.
+        """Scrapes the given URL and returns the HTML content of the page.
+
+        Optional
+        extractRules return deterministic structured data in extracted using CSS
+        selectors, attributes, lists, and nested rules, without an LLM or additional
+        credits. Rules run on the returned HTML after selector and main-content
+        filtering. Send extractRules as a JSON-encoded query parameter. The base request
+        costs 1 credit; requests with browser actions cost 2 credits. A request that
+        hits its timeoutOpts.milliseconds deadline fails with 408 and is not billed,
+        unless timeoutOpts.behavior=return-partial is set — then the page as rendered so
+        far is returned with `finalDOMState: "still-loading"` and billed at the base
+        cost of 1 credit.
 
         Args:
           url: Full URL to scrape (must include http:// or https:// protocol)
@@ -4635,6 +4654,14 @@ async def web_scrape_html(
               Exclusion takes precedence: an element matching both is removed. Examples:
               "nav", "footer", ".ad-banner", "[aria-hidden=true]".
 
+          extract_rules: Optional CSS extraction rules applied to the returned HTML after selector and
+              main-content filtering. Use selector strings ("h1", "a@href") or objects with
+              selector, type (item or list), and output (text, html, @attribute, or nested
+              rules). Text whitespace is normalized; html includes the matched element;
+              attributes are returned as written. Missing items are null and missing lists are
+              empty. CSS only; XPath is not supported. Maximum: 100 fields across 5 levels.
+              Send a JSON-encoded string in the extractRules query parameter.
+
           headers: Optional outbound HTTP headers forwarded only to the target URL, sent as
               deep-object query params such as headers[X-Custom]=value. When provided, caching
               is bypassed: the result is neither read from nor written to cache.
@@ -4698,6 +4725,7 @@ async def web_scrape_html(
                         "actions": actions,
                         "country": country,
                         "exclude_selectors": exclude_selectors,
+                        "extract_rules": extract_rules,
                         "headers": headers,
                         "include_frames": include_frames,
                         "include_selectors": include_selectors,

src/context/dev/types/web_web_scrape_html_params.py

@@ -14,6 +14,10 @@
     "ActionWebScrapeWaitAction",
     "ActionWebScrapePerformAction",
     "ActionWebScrapeScrollAction",
+    "ExtractRules",
+    "ExtractRulesUnionMember1",
+    "ExtractRulesUnionMember1OutputHTMLExtractionRulesExtractRulesUnionMember1OutputHTMLExtractionRulesItem",
+    "ExtractRulesUnionMember1OutputHTMLExtractionRulesExtractRulesUnionMember1OutputHTMLExtractionRulesItemUnionMember1",
     "Pdf",
     "TimeoutOpts",
 ]
@@ -248,6 +252,17 @@ class WebWebScrapeHTMLParams(TypedDict, total=False):
     both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]".
     """
 
+    extract_rules: Annotated[Dict[str, ExtractRules], PropertyInfo(alias="extractRules")]
+    """
+    Optional CSS extraction rules applied to the returned HTML after selector and
+    main-content filtering. Use selector strings ("h1", "a@href") or objects with
+    selector, type (item or list), and output (text, html, @attribute, or nested
+    rules). Text whitespace is normalized; html includes the matched element;
+    attributes are returned as written. Missing items are null and missing lists are
+    empty. CSS only; XPath is not supported. Maximum: 100 fields across 5 levels.
+    Send a JSON-encoded string in the extractRules query parameter.
+    """
+
     headers: Dict[str, str]
     """
     Optional outbound HTTP headers forwarded only to the target URL, sent as
@@ -368,6 +383,39 @@ class ActionWebScrapeScrollAction(TypedDict, total=False):
 Action: TypeAlias = Union[ActionWebScrapeWaitAction, ActionWebScrapePerformAction, ActionWebScrapeScrollAction]
 
 
+class ExtractRulesUnionMember1OutputHTMLExtractionRulesExtractRulesUnionMember1OutputHTMLExtractionRulesItemUnionMember1(
+    TypedDict, total=False
+):
+    selector: Required[str]
+
+    output: Union[Literal["text", "html"], str, object]
+
+    type: Literal["item", "list"]
+
+
+ExtractRulesUnionMember1OutputHTMLExtractionRulesExtractRulesUnionMember1OutputHTMLExtractionRulesItem: TypeAlias = Union[
+    str,
+    ExtractRulesUnionMember1OutputHTMLExtractionRulesExtractRulesUnionMember1OutputHTMLExtractionRulesItemUnionMember1,
+]
+
+
+class ExtractRulesUnionMember1(TypedDict, total=False):
+    selector: Required[str]
+
+    output: Union[
+        Literal["text", "html"],
+        str,
+        Dict[
+            str, ExtractRulesUnionMember1OutputHTMLExtractionRulesExtractRulesUnionMember1OutputHTMLExtractionRulesItem
+        ],
+    ]
+
+    type: Literal["item", "list"]
+
+
+ExtractRules: TypeAlias = Union[str, ExtractRulesUnionMember1]
+
+
 class Pdf(TypedDict, total=False):
     """PDF parsing controls.
 

src/context/dev/types/web_web_scrape_html_response.py

@@ -218,5 +218,13 @@ class WebWebScrapeHTMLResponse(BaseModel):
     afterward.
     """
 
+    extracted: Optional[Dict[str, Union[str, List[Union[str, object, None]], object, None]]] = None
+    """Present only when extractRules is supplied.
+
+    Keys match the requested fields. Values are normalized text, raw attribute
+    strings, outer HTML, nested objects, or lists. Missing items are null; lists
+    with no matches are empty. Rules run on the returned HTML after filtering.
+    """
+
     key_metadata: Optional[KeyMetadata] = None
     """Credit usage, included whenever a valid API key is provided."""

tests/api_resources/test_web.py

@@ -579,6 +579,7 @@ def test_method_web_scrape_html_with_all_params(self, client: ContextDev) -> Non
             ],
             country="de",
             exclude_selectors=["x"],
+            extract_rules={"foo": "x"},
             headers={"foo": "J!"},
             include_frames=True,
             include_selectors=["x"],
@@ -1371,6 +1372,7 @@ async def test_method_web_scrape_html_with_all_params(self, async_client: AsyncC
             ],
             country="de",
             exclude_selectors=["x"],
+            extract_rules={"foo": "x"},
             headers={"foo": "J!"},
             include_frames=True,
             include_selectors=["x"],