fix(scraping): run post-processing on the selected content, not body

crawl4ai/content_scraping_strategy.py

@@ -729,8 +729,10 @@ def _scrap(
             elif content_element is None:
                 content_element = body
 
-            # Replace mermaid SVGs with text before they get stripped
-            for svg in body.xpath('.//svg[starts-with(@id, "mermaid-")]'):
+            # Replace mermaid SVGs with text before they get stripped. Runs on
+            # content_element (see the note above the cleanup block below) so a
+            # selector picks up the placeholder instead of a stale, unmutated copy.
+            for svg in content_element.xpath('.//svg[starts-with(@id, "mermaid-")]'):
                 try:
                     diagram_type = svg.get("aria-roledescription", "diagram")
                     labels = []
@@ -789,14 +791,15 @@ def _scrap(
                 except Exception:
                     pass
 
-            # Remove script and style tags
+            # Remove script and style tags. Same reason as the mermaid pass
+            # above: this must land on content_element, not body.
             for tag in ["style", "link", "meta", "noscript"]:
-                for element in body.xpath(f".//{tag}"):
+                for element in content_element.xpath(f".//{tag}"):
                     if element.getparent() is not None:
                         element.getparent().remove(element)
-                        
+
             # Handle script separately
-            for element in body.xpath(f".//script"):
+            for element in content_element.xpath(f".//script"):
                 parent = element.getparent()
                 if parent is not None:
                     tail = element.tail  # Get the tail text
@@ -824,13 +827,16 @@ def _scrap(
                 )
                 kwargs["exclude_domains"].update(kwargs["exclude_social_media_domains"])
 
-            # Process forms if needed
+            # Process forms if needed. Same reason as the passes above: this
+            # must land on content_element, not body.
             if kwargs.get("remove_forms", False):
-                for form in body.xpath(".//form"):
+                for form in content_element.xpath(".//form"):
                     if form.getparent() is not None:
                         form.getparent().remove(form)
 
-            # Process content
+            # Process content. Link and media collection stays page-wide by
+            # design, so this deliberately reads from body, not content_element,
+            # even when a selector is in play.
             media = {"images": [], "videos": [], "audios": [], "tables": []}
             internal_links_dict = {}
             external_links_dict = {}
@@ -846,7 +852,8 @@ def _scrap(
                 **kwargs,
             )
 
-            # Extract tables using the table extraction strategy if provided
+            # Extract tables using the table extraction strategy if provided.
+            # Page-wide for the same reason as the link/media pass above.
             if 'table' not in excluded_tags:
                 table_extraction = kwargs.get('table_extraction')
                 if table_extraction:
@@ -857,28 +864,35 @@ def _scrap(
                     extracted_tables = table_extraction.extract_tables(body, **kwargs)
                     media["tables"].extend(extracted_tables)
 
+            # Every pass below mutates and then serialises content_element, the
+            # thing that becomes cleaned_html. With a css_selector or
+            # target_elements that is the deep copy severed from body at the
+            # top of this method; without one content_element is body itself,
+            # so the common path is unchanged.
+
             # Handle only_text option
             if kwargs.get("only_text", False):
                 for tag in ONLY_TEXT_ELIGIBLE_TAGS:
-                    for element in body.xpath(f".//{tag}"):
+                    for element in content_element.xpath(f".//{tag}"):
                         if element.text:
                             new_text = lhtml.Element("span")
                             new_text.text = element.text_content()
                             if element.getparent() is not None:
                                 element.getparent().replace(element, new_text)
 
             # Clean base64 images
-            for img in body.xpath(".//img[@src]"):
+            for img in content_element.xpath(".//img[@src]"):
                 src = img.get("src", "")
                 if self.BASE64_PATTERN.match(src):
                     img.set("src", self.BASE64_PATTERN.sub("", src))
 
             # Remove empty elements
-            self.remove_empty_elements_fast(body, 1)
+            self.remove_empty_elements_fast(content_element, 1)
 
             # Remove unneeded attributes
             self.remove_unwanted_attributes_fast(
-                body, keep_data_attributes=kwargs.get("keep_data_attributes", False)
+                content_element,
+                keep_data_attributes=kwargs.get("keep_data_attributes", False),
             )
 
             # Generate output HTML

tests/test_selector_post_processing.py

@@ -0,0 +1,146 @@
+"""Tests for post-processing when css_selector or target_elements is set.
+
+The bug: _scrap() deep-copies the selector match into a new content_element,
+then ran only_text, base64 image cleanup, empty-element removal and attribute
+stripping, plus mermaid SVG replacement, style/link/meta/noscript/script
+removal and form removal, against `body`. The copy is detached from `body`,
+so none of those passes reached the HTML that is serialised into
+cleaned_html: <b> survived only_text, inline style, onclick, data-* and whole
+base64 payloads were emitted verbatim, and a <script>, <style>, <noscript> or
+<form> inside the selection survived into cleaned_html untouched.
+"""
+
+import pytest
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+
+BASE64_SRC = "data:image/png;base64,AAAABBBBCCCCDDDD"
+
+SAMPLE_HTML = f"""
+<html>
+<body>
+    <div class="job" data-tracking="abc" style="color:red" onclick="track()">
+        <p>We are hiring a backend engineer for the platform team.</p>
+        <p>The stack is <b>Postgres</b> and <i>Python</i> in production.</p>
+        <p>Apply through <a href="/apply">our form</a> before the deadline.</p>
+        <img src="{BASE64_SRC}">
+        <div class="tracker"></div>
+        <style>.job {{ color: red; }}</style>
+        <script>track();</script>
+        <noscript>Enable JS to apply.</noscript>
+    </div>
+    <div class="sidebar">
+        <p>More roles are listed on <a href="/jobs">the jobs page</a> today.</p>
+    </div>
+</body>
+</html>
+"""
+
+COMMON = dict(url="raw://test", html=SAMPLE_HTML, only_text=True)
+
+SELECTORS = [
+    pytest.param({}, id="no-selector"),
+    pytest.param({"css_selector": ".job"}, id="css_selector"),
+    pytest.param({"target_elements": [".job"]}, id="target_elements"),
+    pytest.param({"css_selector": "body", "target_elements": [".job"]}, id="both"),
+]
+
+
+@pytest.fixture
+def scraper():
+    return LXMLWebScrapingStrategy()
+
+
+@pytest.mark.parametrize("selector", SELECTORS)
+class TestPostProcessingRunsWithSelector:
+    def test_only_text_unwraps_inline_tags(self, scraper, selector):
+        """only_text should unwrap <b>/<i> whether or not a selector is set."""
+        cleaned = scraper._scrap(**COMMON, **selector)["cleaned_html"]
+        assert "<b>" not in cleaned
+        assert "<i>" not in cleaned
+        assert "Postgres" in cleaned
+        assert "Python" in cleaned
+
+    def test_base64_image_src_is_truncated(self, scraper, selector):
+        """The base64 payload should never reach cleaned_html."""
+        cleaned = scraper._scrap(**COMMON, **selector)["cleaned_html"]
+        assert "base64" not in cleaned
+        assert 'src=""' in cleaned
+
+    def test_empty_elements_are_removed(self, scraper, selector):
+        """The empty <div class="tracker"> should be dropped."""
+        cleaned = scraper._scrap(**COMMON, **selector)["cleaned_html"]
+        assert "tracker" not in cleaned
+
+    def test_unwanted_attributes_are_stripped(self, scraper, selector):
+        """style/onclick/data-* go, class/href stay."""
+        cleaned = scraper._scrap(**COMMON, **selector)["cleaned_html"]
+        assert "style=" not in cleaned
+        assert "onclick=" not in cleaned
+        assert "data-tracking" not in cleaned
+        assert 'class="job"' in cleaned
+        assert 'href="/apply"' in cleaned
+
+    def test_style_script_noscript_are_removed(self, scraper, selector):
+        """style/script/noscript inside the selection must not survive."""
+        cleaned = scraper._scrap(**COMMON, **selector)["cleaned_html"]
+        assert "<style>" not in cleaned
+        assert "<script>" not in cleaned
+        assert "<noscript>" not in cleaned
+        assert "track()" not in cleaned
+        assert "Enable JS to apply" not in cleaned
+
+
+class TestPostProcessingBoundaries:
+    def test_selector_without_match_still_post_processes(self, scraper):
+        """A selector matching nothing falls back to the body, still cleaned."""
+        cleaned = scraper._scrap(**COMMON, css_selector=".nonexistent")["cleaned_html"]
+        assert "<b>" not in cleaned
+        assert "onclick=" not in cleaned
+        assert "the jobs page" in cleaned
+
+    def test_single_inline_element_selected(self, scraper):
+        """The shortest selection: one inline tag that only_text unwraps."""
+        cleaned = scraper._scrap(**COMMON, css_selector="b")["cleaned_html"]
+        assert "<b>" not in cleaned
+        assert "Postgres" in cleaned
+
+    def test_only_text_false_keeps_inline_tags(self, scraper):
+        """Without only_text the inline tags stay, but attributes still go."""
+        cleaned = scraper._scrap(
+            url="raw://test", html=SAMPLE_HTML, css_selector=".job"
+        )["cleaned_html"]
+        assert "<b>Postgres</b>" in cleaned
+        assert "onclick=" not in cleaned
+
+    def test_keep_data_attributes_with_selector(self, scraper):
+        """keep_data_attributes still keeps data-* under a selector."""
+        cleaned = scraper._scrap(
+            **COMMON, css_selector=".job", keep_data_attributes=True
+        )["cleaned_html"]
+        assert 'data-tracking="abc"' in cleaned
+        assert "onclick=" not in cleaned
+
+    def test_selector_still_excludes_other_content(self, scraper):
+        """Post-processing must not widen what the selector selected."""
+        cleaned = scraper._scrap(**COMMON, css_selector=".job")["cleaned_html"]
+        assert "We are hiring" in cleaned
+        assert "the jobs page" not in cleaned
+
+    def test_links_are_collected_from_the_whole_page(self, scraper):
+        """Link extraction stays page-wide, not limited to the selection."""
+        result = scraper._scrap(**COMMON, target_elements=[".job"])
+        hrefs = {link["href"] for link in result["links"]["internal"]}
+        assert any(href.endswith("/apply") for href in hrefs)
+        assert any(href.endswith("/jobs") for href in hrefs)
+
+    def test_form_inside_selection_is_removed(self, scraper):
+        """remove_forms must reach a <form> inside the selected subtree."""
+        html = SAMPLE_HTML.replace(
+            '<div class="tracker"></div>',
+            '<div class="tracker"></div><form><input name="apply"></form>',
+        )
+        cleaned = scraper._scrap(
+            url="raw://test", html=html, css_selector=".job", remove_forms=True
+        )["cleaned_html"]
+        assert "<form>" not in cleaned
+        assert "We are hiring" in cleaned