feat: add optional headers to request

Author: PeriniM

scrapegraph-py/examples/optional_headers_example.py

@@ -0,0 +1,28 @@
+from scrapegraph_py import Client
+from scrapegraph_py.logger import sgai_logger
+
+sgai_logger.set_logging(level="INFO")
+
+# Initialize the client with explicit API key
+sgai_client = Client(api_key="your-api-key-here")
+
+# SmartScraper request
+response = sgai_client.smartscraper(
+    website_url="https://example.com",
+    user_prompt="Extract the main heading, description, and summary of the webpage",
+    headers={
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+        "Accept-Encoding": "gzip, deflate, br",
+        "Connection": "keep-alive",
+        "Upgrade-Insecure-Requests": "1",
+    },
+)
+
+
+# Print the response
+print(f"Request ID: {response['request_id']}")
+print(f"Result: {response['result']}")
+
+sgai_client.close()

scrapegraph-py/scrapegraph_py/async_client.py

@@ -132,11 +132,15 @@ async def _make_request(self, method: str, url: str, **kwargs) -> Any:
                 logger.info(f"⏳ Waiting {retry_delay}s before retry {attempt + 2}")
                 await asyncio.sleep(retry_delay)
 
-    async def markdownify(self, website_url: str):
+    async def markdownify(
+        self, website_url: str, headers: Optional[dict[str, str]] = None
+    ):
         """Send a markdownify request"""
         logger.info(f"🔍 Starting markdownify request for {website_url}")
+        if headers:
+            logger.debug("🔧 Using custom headers")
 
-        request = MarkdownifyRequest(website_url=website_url)
+        request = MarkdownifyRequest(website_url=website_url, headers=headers)
         logger.debug("✅ Request validation passed")
 
         result = await self._make_request(
@@ -164,6 +168,7 @@ async def smartscraper(
         user_prompt: str,
         website_url: Optional[str] = None,
         website_html: Optional[str] = None,
+        headers: Optional[dict[str, str]] = None,
         output_schema: Optional[BaseModel] = None,
     ):
         """Send a smartscraper request"""
@@ -172,11 +177,14 @@ async def smartscraper(
             logger.debug(f"🌐 URL: {website_url}")
         if website_html:
             logger.debug("📄 Using provided HTML content")
+        if headers:
+            logger.debug("🔧 Using custom headers")
         logger.debug(f"📝 Prompt: {user_prompt}")
 
         request = SmartScraperRequest(
             website_url=website_url,
             website_html=website_html,
+            headers=headers,
             user_prompt=user_prompt,
             output_schema=output_schema,
         )

scrapegraph-py/scrapegraph_py/client.py

@@ -144,11 +144,13 @@ def _make_request(self, method: str, url: str, **kwargs) -> Any:
             logger.error(f"🔴 Connection Error: {str(e)}")
             raise ConnectionError(f"Failed to connect to API: {str(e)}")
 
-    def markdownify(self, website_url: str):
+    def markdownify(self, website_url: str, headers: Optional[dict[str, str]] = None):
         """Send a markdownify request"""
         logger.info(f"🔍 Starting markdownify request for {website_url}")
+        if headers:
+            logger.debug("🔧 Using custom headers")
 
-        request = MarkdownifyRequest(website_url=website_url)
+        request = MarkdownifyRequest(website_url=website_url, headers=headers)
         logger.debug("✅ Request validation passed")
 
         result = self._make_request(
@@ -174,6 +176,7 @@ def smartscraper(
         user_prompt: str,
         website_url: Optional[str] = None,
         website_html: Optional[str] = None,
+        headers: Optional[dict[str, str]] = None,
         output_schema: Optional[BaseModel] = None,
     ):
         """Send a smartscraper request"""
@@ -182,11 +185,14 @@ def smartscraper(
             logger.debug(f"🌐 URL: {website_url}")
         if website_html:
             logger.debug("📄 Using provided HTML content")
+        if headers:
+            logger.debug("🔧 Using custom headers")
         logger.debug(f"📝 Prompt: {user_prompt}")
 
         request = SmartScraperRequest(
             website_url=website_url,
             website_html=website_html,
+            headers=headers,
             user_prompt=user_prompt,
             output_schema=output_schema,
         )

scrapegraph-py/scrapegraph_py/models/markdownify.py

@@ -1,12 +1,21 @@
 # Models for markdownify endpoint
 
+from typing import Optional
 from uuid import UUID
 
 from pydantic import BaseModel, Field, model_validator
 
 
 class MarkdownifyRequest(BaseModel):
     website_url: str = Field(..., example="https://scrapegraphai.com/")
+    headers: Optional[dict[str, str]] = Field(
+        None,
+        example={
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+            "Cookie": "cookie1=value1; cookie2=value2",
+        },
+        description="Optional headers to send with the request, including cookies and user agent",
+    )
 
     @model_validator(mode="after")
     def validate_url(self) -> "MarkdownifyRequest":

scrapegraph-py/scrapegraph_py/models/smartscraper.py

@@ -20,6 +20,14 @@ class SmartScraperRequest(BaseModel):
         example="<html><body><h1>Title</h1><p>Content</p></body></html>",
         description="HTML content, maximum size 2MB",
     )
+    headers: Optional[dict[str, str]] = Field(
+        None,
+        example={
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+            "Cookie": "cookie1=value1; cookie2=value2",
+        },
+        description="Optional headers to send with the request, including cookies and user agent",
+    )
     output_schema: Optional[Type[BaseModel]] = None
 
     @model_validator(mode="after")