From e4ddcd33152932aee3593f625a05c3f39538b4ea Mon Sep 17 00:00:00 2001 From: "user.mail" Date: Tue, 28 Jul 2026 16:12:07 +0300 Subject: [PATCH 1/3] add ScraperCore: dedupe construction across collect + search scrapers The 7 search classes hand-copied the base's __init__ and context manager; they now inherit a shared core. Amazon/ChatGPT/LinkedIn search gain the standard env-token fallback. No public API or behavior removed. --- src/brightdata/scrapers/amazon/search.py | 26 ++--- src/brightdata/scrapers/base.py | 113 +++++++++++++------- src/brightdata/scrapers/chatgpt/search.py | 27 ++--- src/brightdata/scrapers/instagram/search.py | 50 +-------- src/brightdata/scrapers/linkedin/search.py | 27 ++--- src/brightdata/scrapers/pinterest/search.py | 50 +-------- src/brightdata/scrapers/tiktok/search.py | 50 +-------- src/brightdata/scrapers/youtube/search.py | 49 +-------- tests/unit/test_scraper_core.py | 105 ++++++++++++++++++ 9 files changed, 214 insertions(+), 283 deletions(-) create mode 100644 tests/unit/test_scraper_core.py diff --git a/src/brightdata/scrapers/amazon/search.py b/src/brightdata/scrapers/amazon/search.py index 955ec6b..74b2a13 100644 --- a/src/brightdata/scrapers/amazon/search.py +++ b/src/brightdata/scrapers/amazon/search.py @@ -11,16 +11,14 @@ import asyncio from typing import Union, List, Optional, Dict, Any -from ...core.engine import AsyncEngine from ...models import ScrapeResult from ...exceptions import ValidationError from ...utils.function_detection import get_caller_function_name from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD -from ..api_client import DatasetAPIClient -from ..workflow import WorkflowExecutor +from ..base import ScraperCore -class AmazonSearchScraper: +class AmazonSearchScraper(ScraperCore): """ Amazon Search Scraper for parameter-based discovery. @@ -39,22 +37,12 @@ class AmazonSearchScraper: # Amazon dataset IDs DATASET_ID_PRODUCTS_SEARCH = "gd_lwdb4vjm1ehb499uxs" # Amazon Products Search (15.84M records) - def __init__(self, bearer_token: str, engine: Optional[AsyncEngine] = None): - """ - Initialize Amazon search scraper. + # Platform configuration (consumed by ScraperCore.__init__) + PLATFORM_NAME = "amazon" + COST_PER_RECORD = DEFAULT_COST_PER_RECORD - Args: - bearer_token: Bright Data API token - engine: Optional AsyncEngine instance (reused from client) - """ - self.bearer_token = bearer_token - self.engine = engine if engine is not None else AsyncEngine(bearer_token) - self.api_client = DatasetAPIClient(self.engine) - self.workflow_executor = WorkflowExecutor( - api_client=self.api_client, - platform_name="amazon", - cost_per_record=DEFAULT_COST_PER_RECORD, - ) + # Construction (token/engine/api_client/workflow_executor) and async + # context-manager support are inherited from ScraperCore. # ============================================================================ # PRODUCTS SEARCH (by keyword + filters) diff --git a/src/brightdata/scrapers/base.py b/src/brightdata/scrapers/base.py index e9b22c5..c76b86f 100644 --- a/src/brightdata/scrapers/base.py +++ b/src/brightdata/scrapers/base.py @@ -32,7 +32,77 @@ from .job import ScrapeJob -class BaseWebScraper(ABC): +class ScraperCore: + """ + Shared construction core for all scraper classes (collect + search). + + Owns what every scraper needs regardless of its verbs: bearer-token + resolution (parameter or BRIGHTDATA_API_TOKEN), engine reuse-or-create, + DatasetAPIClient, WorkflowExecutor, and async context-manager support + for standalone usage. + + BaseWebScraper builds the URL-scrape workflow on top of this core; the + per-platform search classes (parameter-based discovery) inherit the core + directly, since they have no single DATASET_ID or generic scrape method. + """ + + PLATFORM_NAME: str = "" + MIN_POLL_TIMEOUT: int = DEFAULT_MIN_POLL_TIMEOUT + COST_PER_RECORD: float = DEFAULT_COST_PER_RECORD + + def __init__(self, bearer_token: Optional[str] = None, engine: Optional[AsyncEngine] = None): + """ + Initialize scraper core. + + Args: + bearer_token: Bright Data API token. If None, loads from environment. + engine: Optional AsyncEngine instance. If provided, reuses the existing engine + (recommended when using via client to share connection pool and rate limiter). + If None, creates a new engine (for standalone usage). + + Raises: + ValidationError: If token not provided and not in environment + """ + self.bearer_token = bearer_token or os.getenv("BRIGHTDATA_API_TOKEN") + if not self.bearer_token: + raise ValidationError( + f"Bearer token required for {self.PLATFORM_NAME or 'scraper'}. " + f"Provide bearer_token parameter or set BRIGHTDATA_API_TOKEN environment variable." + ) + + # Reuse engine if provided (for resource efficiency), otherwise create new one + self.engine = engine if engine is not None else AsyncEngine(self.bearer_token) + self.api_client = DatasetAPIClient(self.engine) + self.workflow_executor = WorkflowExecutor( + api_client=self.api_client, + platform_name=self.PLATFORM_NAME or None, + cost_per_record=self.COST_PER_RECORD, + ) + + # ============================================================================ + # CONTEXT MANAGER SUPPORT (for standalone usage) + # ============================================================================ + + async def __aenter__(self): + """ + Async context manager entry for standalone scraper usage. + + When using a scraper directly (not through BrightDataClient), + use the context manager to ensure proper engine lifecycle management. + + Example: + >>> async with AmazonScraper(token="...") as scraper: + ... result = await scraper.products(url) + """ + await self.engine.__aenter__() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit - cleanup engine.""" + await self.engine.__aexit__(exc_type, exc_val, exc_tb) + + +class BaseWebScraper(ScraperCore, ABC): """ Base class for all platform-specific scrapers. @@ -60,9 +130,6 @@ class BaseWebScraper(ABC): """ DATASET_ID: str = "" - PLATFORM_NAME: str = "" - MIN_POLL_TIMEOUT: int = DEFAULT_MIN_POLL_TIMEOUT - COST_PER_RECORD: float = DEFAULT_COST_PER_RECORD def __init__(self, bearer_token: Optional[str] = None, engine: Optional[AsyncEngine] = None): """ @@ -77,21 +144,7 @@ def __init__(self, bearer_token: Optional[str] = None, engine: Optional[AsyncEng Raises: ValidationError: If token not provided and not in environment """ - self.bearer_token = bearer_token or os.getenv("BRIGHTDATA_API_TOKEN") - if not self.bearer_token: - raise ValidationError( - f"Bearer token required for {self.PLATFORM_NAME or 'scraper'}. " - f"Provide bearer_token parameter or set BRIGHTDATA_API_TOKEN environment variable." - ) - - # Reuse engine if provided (for resource efficiency), otherwise create new one - self.engine = engine if engine is not None else AsyncEngine(self.bearer_token) - self.api_client = DatasetAPIClient(self.engine) - self.workflow_executor = WorkflowExecutor( - api_client=self.api_client, - platform_name=self.PLATFORM_NAME or None, - cost_per_record=self.COST_PER_RECORD, - ) + super().__init__(bearer_token=bearer_token, engine=engine) if not self.DATASET_ID: raise NotImplementedError( @@ -489,28 +542,6 @@ async def to_result( data_fetched_at=datetime.now(timezone.utc), ) - # ============================================================================ - # CONTEXT MANAGER SUPPORT (for standalone usage) - # ============================================================================ - - async def __aenter__(self): - """ - Async context manager entry for standalone scraper usage. - - When using a scraper directly (not through BrightDataClient), - use the context manager to ensure proper engine lifecycle management. - - Example: - >>> async with AmazonScraper(token="...") as scraper: - ... result = await scraper.products(url) - """ - await self.engine.__aenter__() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Async context manager exit - cleanup engine.""" - await self.engine.__aexit__(exc_type, exc_val, exc_tb) - def __repr__(self) -> str: """String representation for debugging.""" platform = self.PLATFORM_NAME or self.__class__.__name__ diff --git a/src/brightdata/scrapers/chatgpt/search.py b/src/brightdata/scrapers/chatgpt/search.py index a2a9548..a214085 100644 --- a/src/brightdata/scrapers/chatgpt/search.py +++ b/src/brightdata/scrapers/chatgpt/search.py @@ -12,16 +12,14 @@ import asyncio from typing import Union, List, Optional, Dict, Any -from ...core.engine import AsyncEngine from ...models import ScrapeResult from ...exceptions import ValidationError from ...utils.function_detection import get_caller_function_name from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT, COST_PER_RECORD_CHATGPT -from ..api_client import DatasetAPIClient -from ..workflow import WorkflowExecutor +from ..base import ScraperCore -class ChatGPTSearchService: +class ChatGPTSearchService(ScraperCore): """ ChatGPT Search Service for prompt-based discovery. @@ -50,23 +48,12 @@ class ChatGPTSearchService: DATASET_ID = "gd_m7aof0k82r803d5bjm" # ChatGPT dataset - def __init__(self, bearer_token: str, engine: Optional[AsyncEngine] = None): - """ - Initialize ChatGPT search service. + # Platform configuration (consumed by ScraperCore.__init__) + PLATFORM_NAME = "chatgpt" + COST_PER_RECORD = COST_PER_RECORD_CHATGPT - Args: - bearer_token: Bright Data API token - engine: Optional AsyncEngine instance. If not provided, creates a new one. - Allows dependency injection for testing and flexibility. - """ - self.bearer_token = bearer_token - self.engine = engine if engine is not None else AsyncEngine(bearer_token) - self.api_client = DatasetAPIClient(self.engine) - self.workflow_executor = WorkflowExecutor( - api_client=self.api_client, - platform_name="chatgpt", - cost_per_record=COST_PER_RECORD_CHATGPT, - ) + # Construction (token/engine/api_client/workflow_executor) and async + # context-manager support are inherited from ScraperCore. # ============================================================================ # CHATGPT PROMPT DISCOVERY diff --git a/src/brightdata/scrapers/instagram/search.py b/src/brightdata/scrapers/instagram/search.py index bc80aa0..8f5a882 100644 --- a/src/brightdata/scrapers/instagram/search.py +++ b/src/brightdata/scrapers/instagram/search.py @@ -8,14 +8,10 @@ """ import asyncio -import os from typing import List, Dict, Any, Optional, Union -from ..api_client import DatasetAPIClient -from ..workflow import WorkflowExecutor -from ...core.engine import AsyncEngine +from ..base import ScraperCore from ...models import ScrapeResult -from ...exceptions import ValidationError from ...constants import ( COST_PER_RECORD_INSTAGRAM, DEFAULT_TIMEOUT_SHORT, @@ -25,7 +21,7 @@ from ...utils.function_detection import get_caller_function_name -class InstagramSearchScraper: +class InstagramSearchScraper(ScraperCore): """ Instagram scraper for parameter-based content discovery. @@ -52,46 +48,8 @@ class InstagramSearchScraper: MIN_POLL_TIMEOUT = DEFAULT_TIMEOUT_SHORT COST_PER_RECORD = COST_PER_RECORD_INSTAGRAM - def __init__( - self, - bearer_token: Optional[str] = None, - engine: Optional[AsyncEngine] = None, - ): - """ - Initialize Instagram search scraper. - - Args: - bearer_token: Bright Data API token. If None, loads from environment. - engine: Optional AsyncEngine instance for connection reuse. - """ - self.bearer_token = bearer_token or os.getenv("BRIGHTDATA_API_TOKEN") - if not self.bearer_token: - raise ValidationError( - "Bearer token required for Instagram search. " - "Provide bearer_token parameter or set BRIGHTDATA_API_TOKEN environment variable." - ) - - # Reuse engine if provided, otherwise create new - self.engine = engine if engine is not None else AsyncEngine(self.bearer_token) - self.api_client = DatasetAPIClient(self.engine) - self.workflow_executor = WorkflowExecutor( - api_client=self.api_client, - platform_name=self.PLATFORM_NAME, - cost_per_record=self.COST_PER_RECORD, - ) - - # ============================================================================ - # CONTEXT MANAGER SUPPORT - # ============================================================================ - - async def __aenter__(self): - """Async context manager entry.""" - await self.engine.__aenter__() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Async context manager exit.""" - await self.engine.__aexit__(exc_type, exc_val, exc_tb) + # Construction (token/engine/api_client/workflow_executor) and async + # context-manager support are inherited from ScraperCore. # ============================================================================ # INTERNAL HELPERS diff --git a/src/brightdata/scrapers/linkedin/search.py b/src/brightdata/scrapers/linkedin/search.py index 249e4ae..f72261e 100644 --- a/src/brightdata/scrapers/linkedin/search.py +++ b/src/brightdata/scrapers/linkedin/search.py @@ -13,16 +13,14 @@ import asyncio from typing import Union, List, Optional, Dict, Any -from ...core.engine import AsyncEngine from ...models import ScrapeResult from ...exceptions import ValidationError from ...utils.function_detection import get_caller_function_name from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT, COST_PER_RECORD_LINKEDIN -from ..api_client import DatasetAPIClient -from ..workflow import WorkflowExecutor +from ..base import ScraperCore -class LinkedInSearchScraper: +class LinkedInSearchScraper(ScraperCore): """ LinkedIn Search Scraper for parameter-based discovery. @@ -55,23 +53,12 @@ class LinkedInSearchScraper: DATASET_ID_JOBS = "gd_lpfll7v5hcqtkxl6l" # URL-based job scraping DATASET_ID_JOBS_DISCOVERY = "gd_m487ihp32jtc4ujg45" # Keyword/location discovery - def __init__(self, bearer_token: str, engine: Optional[AsyncEngine] = None): - """ - Initialize LinkedIn search scraper. + # Platform configuration (consumed by ScraperCore.__init__) + PLATFORM_NAME = "linkedin" + COST_PER_RECORD = COST_PER_RECORD_LINKEDIN - Args: - bearer_token: Bright Data API token - engine: Optional AsyncEngine instance. If not provided, creates a new one. - Allows dependency injection for testing and flexibility. - """ - self.bearer_token = bearer_token - self.engine = engine if engine is not None else AsyncEngine(bearer_token) - self.api_client = DatasetAPIClient(self.engine) - self.workflow_executor = WorkflowExecutor( - api_client=self.api_client, - platform_name="linkedin", - cost_per_record=COST_PER_RECORD_LINKEDIN, - ) + # Construction (token/engine/api_client/workflow_executor) and async + # context-manager support are inherited from ScraperCore. # ============================================================================ # POSTS DISCOVERY (by profile + date range) diff --git a/src/brightdata/scrapers/pinterest/search.py b/src/brightdata/scrapers/pinterest/search.py index 3727258..c116833 100644 --- a/src/brightdata/scrapers/pinterest/search.py +++ b/src/brightdata/scrapers/pinterest/search.py @@ -16,14 +16,10 @@ """ import asyncio -import os from typing import List, Dict, Any, Optional, Union -from ..api_client import DatasetAPIClient -from ..workflow import WorkflowExecutor -from ...core.engine import AsyncEngine +from ..base import ScraperCore from ...models import ScrapeResult -from ...exceptions import ValidationError from ...constants import ( DEFAULT_COST_PER_RECORD, DEFAULT_TIMEOUT_MEDIUM, @@ -32,7 +28,7 @@ from ...utils.function_detection import get_caller_function_name -class PinterestSearchScraper: +class PinterestSearchScraper(ScraperCore): """ Pinterest scraper for parameter-based content discovery. @@ -67,46 +63,8 @@ class PinterestSearchScraper: MIN_POLL_TIMEOUT = DEFAULT_TIMEOUT_MEDIUM COST_PER_RECORD = DEFAULT_COST_PER_RECORD - def __init__( - self, - bearer_token: Optional[str] = None, - engine: Optional[AsyncEngine] = None, - ): - """ - Initialize Pinterest search scraper. - - Args: - bearer_token: Bright Data API token. If None, loads from environment. - engine: Optional AsyncEngine instance for connection reuse. - """ - self.bearer_token = bearer_token or os.getenv("BRIGHTDATA_API_TOKEN") - if not self.bearer_token: - raise ValidationError( - "Bearer token required for Pinterest search. " - "Provide bearer_token parameter or set BRIGHTDATA_API_TOKEN environment variable." - ) - - # Reuse engine if provided, otherwise create new - self.engine = engine if engine is not None else AsyncEngine(self.bearer_token) - self.api_client = DatasetAPIClient(self.engine) - self.workflow_executor = WorkflowExecutor( - api_client=self.api_client, - platform_name=self.PLATFORM_NAME, - cost_per_record=self.COST_PER_RECORD, - ) - - # ============================================================================ - # CONTEXT MANAGER SUPPORT - # ============================================================================ - - async def __aenter__(self): - """Async context manager entry.""" - await self.engine.__aenter__() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Async context manager exit.""" - await self.engine.__aexit__(exc_type, exc_val, exc_tb) + # Construction (token/engine/api_client/workflow_executor) and async + # context-manager support are inherited from ScraperCore. # ============================================================================ # INTERNAL HELPERS diff --git a/src/brightdata/scrapers/tiktok/search.py b/src/brightdata/scrapers/tiktok/search.py index bf697d0..b7eede2 100644 --- a/src/brightdata/scrapers/tiktok/search.py +++ b/src/brightdata/scrapers/tiktok/search.py @@ -16,14 +16,10 @@ """ import asyncio -import os from typing import List, Dict, Any, Optional, Union -from ..api_client import DatasetAPIClient -from ..workflow import WorkflowExecutor -from ...core.engine import AsyncEngine +from ..base import ScraperCore from ...models import ScrapeResult -from ...exceptions import ValidationError from ...constants import ( COST_PER_RECORD_TIKTOK, DEFAULT_TIMEOUT_MEDIUM, @@ -32,7 +28,7 @@ from ...utils.function_detection import get_caller_function_name -class TikTokSearchScraper: +class TikTokSearchScraper(ScraperCore): """ TikTok scraper for parameter-based content discovery. @@ -69,46 +65,8 @@ class TikTokSearchScraper: MIN_POLL_TIMEOUT = DEFAULT_TIMEOUT_MEDIUM COST_PER_RECORD = COST_PER_RECORD_TIKTOK - def __init__( - self, - bearer_token: Optional[str] = None, - engine: Optional[AsyncEngine] = None, - ): - """ - Initialize TikTok search scraper. - - Args: - bearer_token: Bright Data API token. If None, loads from environment. - engine: Optional AsyncEngine instance for connection reuse. - """ - self.bearer_token = bearer_token or os.getenv("BRIGHTDATA_API_TOKEN") - if not self.bearer_token: - raise ValidationError( - "Bearer token required for TikTok search. " - "Provide bearer_token parameter or set BRIGHTDATA_API_TOKEN environment variable." - ) - - # Reuse engine if provided, otherwise create new - self.engine = engine if engine is not None else AsyncEngine(self.bearer_token) - self.api_client = DatasetAPIClient(self.engine) - self.workflow_executor = WorkflowExecutor( - api_client=self.api_client, - platform_name=self.PLATFORM_NAME, - cost_per_record=self.COST_PER_RECORD, - ) - - # ============================================================================ - # CONTEXT MANAGER SUPPORT - # ============================================================================ - - async def __aenter__(self): - """Async context manager entry.""" - await self.engine.__aenter__() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Async context manager exit.""" - await self.engine.__aexit__(exc_type, exc_val, exc_tb) + # Construction (token/engine/api_client/workflow_executor) and async + # context-manager support are inherited from ScraperCore. # ============================================================================ # INTERNAL HELPERS diff --git a/src/brightdata/scrapers/youtube/search.py b/src/brightdata/scrapers/youtube/search.py index 8532db5..4869770 100644 --- a/src/brightdata/scrapers/youtube/search.py +++ b/src/brightdata/scrapers/youtube/search.py @@ -19,14 +19,10 @@ """ import asyncio -import os from typing import List, Dict, Any, Optional, Union -from ..api_client import DatasetAPIClient -from ..workflow import WorkflowExecutor -from ...core.engine import AsyncEngine +from ..base import ScraperCore from ...models import ScrapeResult -from ...exceptions import ValidationError from ...constants import ( COST_PER_RECORD_YOUTUBE, DEFAULT_TIMEOUT_MEDIUM, @@ -35,7 +31,7 @@ from ...utils.function_detection import get_caller_function_name -class YouTubeSearchScraper: +class YouTubeSearchScraper(ScraperCore): """ YouTube scraper for parameter-based content discovery. @@ -73,45 +69,8 @@ class YouTubeSearchScraper: MIN_POLL_TIMEOUT = DEFAULT_TIMEOUT_MEDIUM COST_PER_RECORD = COST_PER_RECORD_YOUTUBE - def __init__( - self, - bearer_token: Optional[str] = None, - engine: Optional[AsyncEngine] = None, - ): - """ - Initialize YouTube search scraper. - - Args: - bearer_token: Bright Data API token. If None, loads from environment. - engine: Optional AsyncEngine instance for connection reuse. - """ - self.bearer_token = bearer_token or os.getenv("BRIGHTDATA_API_TOKEN") - if not self.bearer_token: - raise ValidationError( - "Bearer token required for YouTube search. " - "Provide bearer_token parameter or set BRIGHTDATA_API_TOKEN environment variable." - ) - - self.engine = engine if engine is not None else AsyncEngine(self.bearer_token) - self.api_client = DatasetAPIClient(self.engine) - self.workflow_executor = WorkflowExecutor( - api_client=self.api_client, - platform_name=self.PLATFORM_NAME, - cost_per_record=self.COST_PER_RECORD, - ) - - # ============================================================================ - # CONTEXT MANAGER SUPPORT - # ============================================================================ - - async def __aenter__(self): - """Async context manager entry.""" - await self.engine.__aenter__() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Async context manager exit.""" - await self.engine.__aexit__(exc_type, exc_val, exc_tb) + # Construction (token/engine/api_client/workflow_executor) and async + # context-manager support are inherited from ScraperCore. # ============================================================================ # INTERNAL HELPERS diff --git a/tests/unit/test_scraper_core.py b/tests/unit/test_scraper_core.py new file mode 100644 index 0000000..c074d15 --- /dev/null +++ b/tests/unit/test_scraper_core.py @@ -0,0 +1,105 @@ +""" +Tests for ScraperCore — the shared construction core for collect + search scrapers. + +Before ScraperCore, the 7 search classes each hand-copied the same __init__ +(token resolution, engine reuse-or-create, DatasetAPIClient, WorkflowExecutor) +and had drifted (Amazon/ChatGPT/LinkedIn lacked the env-token fallback). These +tests pin the consolidated behavior. +""" + +import pytest +from unittest.mock import MagicMock + +from brightdata.exceptions import ValidationError +from brightdata.scrapers.base import BaseWebScraper, ScraperCore +from brightdata.scrapers.amazon.search import AmazonSearchScraper +from brightdata.scrapers.chatgpt.search import ChatGPTSearchService +from brightdata.scrapers.instagram.search import InstagramSearchScraper +from brightdata.scrapers.linkedin.search import LinkedInSearchScraper +from brightdata.scrapers.pinterest.search import PinterestSearchScraper +from brightdata.scrapers.tiktok.search import TikTokSearchScraper +from brightdata.scrapers.youtube.search import YouTubeSearchScraper + +SEARCH_CLASSES = [ + AmazonSearchScraper, + ChatGPTSearchService, + InstagramSearchScraper, + LinkedInSearchScraper, + PinterestSearchScraper, + TikTokSearchScraper, + YouTubeSearchScraper, +] + +EXPECTED_PLATFORMS = { + AmazonSearchScraper: "amazon", + ChatGPTSearchService: "chatgpt", + InstagramSearchScraper: "instagram", + LinkedInSearchScraper: "linkedin", + PinterestSearchScraper: "pinterest", + TikTokSearchScraper: "tiktok", + YouTubeSearchScraper: "youtube", +} + +TOKEN = "x" * 20 + + +class TestHierarchy: + def test_search_classes_inherit_scraper_core(self): + for cls in SEARCH_CLASSES: + assert issubclass(cls, ScraperCore), cls.__name__ + + def test_base_web_scraper_inherits_scraper_core(self): + assert issubclass(BaseWebScraper, ScraperCore) + + def test_search_classes_do_not_gain_scrape_surface(self): + """Search classes take the core only — not BaseWebScraper's URL-scrape API.""" + for cls in SEARCH_CLASSES: + assert not issubclass(cls, BaseWebScraper), cls.__name__ + assert not hasattr(cls, "scrape_async"), cls.__name__ + + +class TestConstruction: + @pytest.mark.parametrize("cls", SEARCH_CLASSES, ids=lambda c: c.__name__) + def test_wires_engine_api_client_workflow(self, cls): + engine = MagicMock() + s = cls(bearer_token=TOKEN, engine=engine) + assert s.bearer_token == TOKEN + assert s.engine is engine + assert s.api_client.engine is engine + assert s.workflow_executor.api_client is s.api_client + assert s.workflow_executor.platform_name == EXPECTED_PLATFORMS[cls] + assert s.workflow_executor.cost_per_record == cls.COST_PER_RECORD + + @pytest.mark.parametrize("cls", SEARCH_CLASSES, ids=lambda c: c.__name__) + def test_env_token_fallback(self, cls, monkeypatch): + """All search classes resolve BRIGHTDATA_API_TOKEN (Amazon/ChatGPT/LinkedIn + previously required an explicit bearer_token).""" + monkeypatch.setenv("BRIGHTDATA_API_TOKEN", "tok_from_env_12345") + s = cls(engine=MagicMock()) + assert s.bearer_token == "tok_from_env_12345" + + @pytest.mark.parametrize("cls", SEARCH_CLASSES, ids=lambda c: c.__name__) + def test_missing_token_raises(self, cls, monkeypatch): + monkeypatch.delenv("BRIGHTDATA_API_TOKEN", raising=False) + with pytest.raises(ValidationError): + cls(engine=MagicMock()) + + def test_positional_token_still_works(self): + """Amazon/ChatGPT/LinkedIn callers pass bearer_token positionally.""" + for cls in (AmazonSearchScraper, ChatGPTSearchService, LinkedInSearchScraper): + assert cls(TOKEN).bearer_token == TOKEN + + +class TestContextManager: + @pytest.mark.parametrize("cls", SEARCH_CLASSES, ids=lambda c: c.__name__) + async def test_aenter_aexit_drive_engine(self, cls): + engine = MagicMock() + + async def _noop(*a, **k): + return engine + + engine.__aenter__ = _noop + engine.__aexit__ = _noop + s = cls(bearer_token=TOKEN, engine=engine) + assert await s.__aenter__() is s + await s.__aexit__(None, None, None) From 1a605987a13b3a419dfdead0c832e40168dc68ba Mon Sep 17 00:00:00 2001 From: "user.mail" Date: Tue, 28 Jul 2026 16:20:46 +0300 Subject: [PATCH 2/3] resolve credentials from the CLI store + report auth source in user-agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Token resolution: param → env (BRIGHTDATA_API_TOKEN, BRIGHTDATA_API_KEY) → ~/.../brightdata-cli/credentials.json (read-only) → actionable error. User-agent is now brightdata-sdk-python/ (auth=param|env|cli_credentials). --- README.md | 3 + src/brightdata/cli_credentials.py | 47 +++++++ src/brightdata/client.py | 41 ++++-- src/brightdata/core/engine.py | 12 +- src/brightdata/sync_client.py | 3 +- tests/unit/test_cli_credentials.py | 206 +++++++++++++++++++++++++++++ tests/unit/test_engine.py | 5 +- 7 files changed, 301 insertions(+), 16 deletions(-) create mode 100644 src/brightdata/cli_credentials.py create mode 100644 tests/unit/test_cli_credentials.py diff --git a/README.md b/README.md index 3b4d3d0..0c45c75 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,9 @@ Get your API Token from the [Bright Data Control Panel](https://brightdata.com/c export BRIGHTDATA_API_TOKEN="your_api_token_here" ``` +**Already logged in with the CLI?** The SDK works with no configuration — it automatically +falls back to the credentials stored by `brightdata login`. + ## Quick Start This SDK is **async-native**. A sync client is also available (see [Sync Client](#sync-client)). diff --git a/src/brightdata/cli_credentials.py b/src/brightdata/cli_credentials.py new file mode 100644 index 0000000..8c9a847 --- /dev/null +++ b/src/brightdata/cli_credentials.py @@ -0,0 +1,47 @@ +""" +Read the API key stored by the Bright Data CLI (`brightdata login`). + +The CLI persists credentials to a well-known per-platform location: + +- Linux: ~/.config/brightdata-cli/credentials.json +- macOS: ~/Library/Application Support/brightdata-cli/credentials.json +- Windows: %APPDATA%\\brightdata-cli\\credentials.json + +File format (all login flows): {"api_key": "KEY"} + +The SDK treats this store as strictly read-only: it never writes into the +brightdata-cli directory and never reads the config.json that lives beside +credentials.json. +""" + +import json +import os +import sys +from pathlib import Path +from typing import Optional + + +def _cli_credentials_path() -> Path: + """Return the platform-specific path of the CLI's credentials.json.""" + if sys.platform == "win32": + base = Path(os.environ.get("APPDATA", str(Path.home() / "AppData" / "Roaming"))) + elif sys.platform == "darwin": + base = Path.home() / "Library" / "Application Support" + else: # linux and others + base = Path.home() / ".config" + return base / "brightdata-cli" / "credentials.json" + + +def read_cli_credentials() -> Optional[str]: + """ + Return the API key stored by `brightdata login`, or None if unavailable. + + Any failure — missing file, malformed JSON, wrong value type, empty key, + no read permission — means "not available" and returns None. This function + never raises and never writes. + """ + try: + key = json.loads(_cli_credentials_path().read_text()).get("api_key") + return key.strip() if isinstance(key, str) and key.strip() else None + except Exception: + return None # missing file, bad JSON, no permission — all mean "not available" diff --git a/src/brightdata/client.py b/src/brightdata/client.py index 08aae05..71512f9 100644 --- a/src/brightdata/client.py +++ b/src/brightdata/client.py @@ -34,6 +34,7 @@ from .datasets import DatasetsClient from .models import ScrapeResult from .types import AccountInfo +from .cli_credentials import read_cli_credentials from http import HTTPStatus from .exceptions import ValidationError, AuthenticationError, APIError @@ -68,8 +69,9 @@ class BrightDataClient: DEFAULT_WEB_UNLOCKER_ZONE = "sdk_unlocker" DEFAULT_SERP_ZONE = "sdk_serp" - # Environment variable name for API token + # Environment variable names for API token (checked in this order) TOKEN_ENV_VAR = "BRIGHTDATA_API_TOKEN" + TOKEN_ENV_VAR_ALT = "BRIGHTDATA_API_KEY" def __init__( self, @@ -95,8 +97,10 @@ def __init__( Supports loading from .env files (requires python-dotenv package). Args: - token: API token. If None, loads from BRIGHTDATA_API_TOKEN environment variable - (supports .env files via python-dotenv) + token: API token. If None, loads from the BRIGHTDATA_API_TOKEN / + BRIGHTDATA_API_KEY environment variables (supports .env files via + python-dotenv), then falls back to the credentials stored by the + Bright Data CLI (`brightdata login`) timeout: Default timeout in seconds for all requests (default: 30) web_unlocker_zone: Zone name for web unlocker (default: "sdk_unlocker") serp_zone: Zone name for SERP API (default: "sdk_serp") @@ -129,7 +133,7 @@ def __init__( ... validate_token=True ... ) """ - self.token = self._load_token(token) + self.token, self.auth_source = self._load_token(token) self.timeout = timeout self.web_unlocker_zone = web_unlocker_zone or self.DEFAULT_WEB_UNLOCKER_ZONE self.serp_zone = serp_zone or self.DEFAULT_SERP_ZONE @@ -146,6 +150,7 @@ def __init__( rate_period=rate_period, ssl_verify=ssl_verify, ssl_ca_cert=ssl_ca_cert, + auth_source=self.auth_source, ) self._scrape_service: Optional[ScrapeService] = None @@ -177,9 +182,13 @@ def _ensure_initialized(self) -> None: "Use: async with BrightDataClient() as client: ..." ) - def _load_token(self, token: Optional[str]) -> str: + def _load_token(self, token: Optional[str]) -> tuple: """ - Load token from parameter or environment variable. + Resolve the API token and record where it came from. + + Resolution order: explicit parameter → environment variables + (BRIGHTDATA_API_TOKEN, then BRIGHTDATA_API_KEY) → the credentials + stored by the Bright Data CLI (`brightdata login`). Fails fast with clear error message if no token found. @@ -187,7 +196,9 @@ def _load_token(self, token: Optional[str]) -> str: token: Explicit token (takes precedence) Returns: - Valid token string + Tuple of (token, auth_source) where auth_source is "param", + "env", or "cli_credentials" — reported in the User-Agent so + SDK onboarding is measurable. The token itself is never logged. Raises: ValidationError: If no token found @@ -198,19 +209,25 @@ def _load_token(self, token: Optional[str]) -> str: f"Invalid token format. Token must be a string with at least 10 characters. " f"Got: {type(token).__name__} with length {len(str(token))}" ) - return token.strip() + return token.strip(), "param" - # Try loading from environment variable - env_token = os.getenv(self.TOKEN_ENV_VAR) + # Try loading from environment variables + env_token = os.getenv(self.TOKEN_ENV_VAR) or os.getenv(self.TOKEN_ENV_VAR_ALT) if env_token: - return env_token.strip() + return env_token.strip(), "env" + + # Fall back to the CLI's stored credentials (read-only) + cli_token = read_cli_credentials() + if cli_token: + return cli_token, "cli_credentials" # No token found - fail fast with helpful message raise ValidationError( f"API token required but not found.\n\n" f"Provide token in one of these ways:\n" f" 1. Pass as parameter: BrightDataClient(token='your_token')\n" - f" 2. Set environment variable: {self.TOKEN_ENV_VAR}\n\n" + f" 2. Set environment variable: {self.TOKEN_ENV_VAR}\n" + f" 3. Log in with the Bright Data CLI: brightdata login\n\n" f"Get your API token from: https://brightdata.com/cp/api_keys" ) diff --git a/src/brightdata/core/engine.py b/src/brightdata/core/engine.py index aa54744..32490fa 100644 --- a/src/brightdata/core/engine.py +++ b/src/brightdata/core/engine.py @@ -52,6 +52,7 @@ def __init__( rate_period: float = 1.0, ssl_verify: bool = True, ssl_ca_cert: Optional[str] = None, + auth_source: Optional[str] = None, ): """ Initialize async engine. @@ -66,12 +67,17 @@ def __init__( Set to False for sandbox/proxy environments. ssl_ca_cert: Path to a custom CA certificate bundle file. Use when behind a corporate proxy with its own CA. + auth_source: How the bearer token was obtained ("param", "env", + "cli_credentials"). Reported in the User-Agent for + onboarding metrics; the token itself is never logged. + None (e.g. standalone scraper usage) omits the field. """ self.bearer_token = bearer_token self.timeout = aiohttp.ClientTimeout(total=timeout) self._session: Optional[aiohttp.ClientSession] = None self._ssl_verify = ssl_verify self._ssl_ca_cert = ssl_ca_cert + self._auth_source = auth_source # Store rate limit config (create limiter per event loop in __aenter__) if rate_limit is None: @@ -103,6 +109,10 @@ async def __aenter__(self): ) # Create session with the connector + user_agent = f"brightdata-sdk-python/{__version__}" + if self._auth_source: + user_agent += f" (auth={self._auth_source})" + self._session = aiohttp.ClientSession( connector=connector, trust_env=True, @@ -110,7 +120,7 @@ async def __aenter__(self): headers={ "Authorization": f"Bearer {self.bearer_token}", "Content-Type": "application/json", - "User-Agent": f"brightdata-sdk/{__version__}", + "User-Agent": user_agent, }, ) diff --git a/src/brightdata/sync_client.py b/src/brightdata/sync_client.py index 8e463dc..8a6995a 100644 --- a/src/brightdata/sync_client.py +++ b/src/brightdata/sync_client.py @@ -54,7 +54,8 @@ def __init__( Initialize sync client. Args: - token: Bright Data API token (or set BRIGHTDATA_API_TOKEN env var) + token: Bright Data API token (or set BRIGHTDATA_API_TOKEN env var, or + log in once with the Bright Data CLI: `brightdata login`) timeout: Default request timeout in seconds web_unlocker_zone: Zone name for Web Unlocker API serp_zone: Zone name for SERP API diff --git a/tests/unit/test_cli_credentials.py b/tests/unit/test_cli_credentials.py new file mode 100644 index 0000000..95fec23 --- /dev/null +++ b/tests/unit/test_cli_credentials.py @@ -0,0 +1,206 @@ +""" +Tests for CLI-credential resolution and auth-source reporting. + +Covers: the per-platform credentials path, read_cli_credentials() failure +modes, the client's resolution precedence (param -> env -> CLI store -> +actionable error), and the User-Agent auth field. +""" + +import json +import sys +from pathlib import Path + +import pytest + +import brightdata.client as client_module +from brightdata.client import BrightDataClient +from brightdata.cli_credentials import _cli_credentials_path, read_cli_credentials +from brightdata.core.engine import AsyncEngine +from brightdata.exceptions import ValidationError + +PARAM_TOKEN = "param_token_1234567890" +ENV_TOKEN = "env_token_1234567890" +CLI_TOKEN = "cli_token_1234567890" + + +@pytest.fixture +def no_env(monkeypatch): + """Clear both token env vars (a repo .env may have populated them via dotenv).""" + monkeypatch.delenv("BRIGHTDATA_API_TOKEN", raising=False) + monkeypatch.delenv("BRIGHTDATA_API_KEY", raising=False) + + +@pytest.fixture +def fake_home(tmp_path, monkeypatch): + """Point Path.home() at a temp dir so tests never touch the real CLI store.""" + monkeypatch.setattr(Path, "home", lambda: tmp_path) + return tmp_path + + +def _write_credentials(base: Path, content) -> Path: + cred_dir = base / "brightdata-cli" + cred_dir.mkdir(parents=True, exist_ok=True) + path = cred_dir / "credentials.json" + path.write_text(content if isinstance(content, str) else json.dumps(content)) + return path + + +# --------------------------------------------------------------------------- +# Platform-specific credentials path +# --------------------------------------------------------------------------- + + +class TestCredentialsPath: + def test_darwin(self, fake_home, monkeypatch): + monkeypatch.setattr(sys, "platform", "darwin") + expected = fake_home / "Library" / "Application Support" / "brightdata-cli" + assert _cli_credentials_path() == expected / "credentials.json" + + def test_linux(self, fake_home, monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + expected = fake_home / ".config" / "brightdata-cli" + assert _cli_credentials_path() == expected / "credentials.json" + + def test_win32_with_appdata(self, fake_home, tmp_path, monkeypatch): + monkeypatch.setattr(sys, "platform", "win32") + appdata = tmp_path / "Roaming" + monkeypatch.setenv("APPDATA", str(appdata)) + assert _cli_credentials_path() == appdata / "brightdata-cli" / "credentials.json" + + def test_win32_without_appdata(self, fake_home, monkeypatch): + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.delenv("APPDATA", raising=False) + expected = fake_home / "AppData" / "Roaming" / "brightdata-cli" + assert _cli_credentials_path() == expected / "credentials.json" + + +# --------------------------------------------------------------------------- +# read_cli_credentials — happy path + every failure mode returns None +# --------------------------------------------------------------------------- + + +class TestReadCliCredentials: + @pytest.fixture(autouse=True) + def _linux(self, monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + + def test_valid_file(self, fake_home): + _write_credentials(fake_home / ".config", {"api_key": CLI_TOKEN}) + assert read_cli_credentials() == CLI_TOKEN + + def test_key_is_trimmed(self, fake_home): + _write_credentials(fake_home / ".config", {"api_key": f" {CLI_TOKEN} "}) + assert read_cli_credentials() == CLI_TOKEN + + def test_missing_file(self, fake_home): + assert read_cli_credentials() is None + + def test_malformed_json(self, fake_home): + _write_credentials(fake_home / ".config", "{not json") + assert read_cli_credentials() is None + + def test_missing_api_key_field(self, fake_home): + _write_credentials(fake_home / ".config", {"other": "x"}) + assert read_cli_credentials() is None + + def test_non_string_api_key(self, fake_home): + _write_credentials(fake_home / ".config", {"api_key": 12345}) + assert read_cli_credentials() is None + + def test_empty_api_key(self, fake_home): + _write_credentials(fake_home / ".config", {"api_key": " "}) + assert read_cli_credentials() is None + + def test_config_json_is_never_read(self, fake_home): + """Only credentials.json is consulted — config.json beside it is ignored.""" + cred_dir = fake_home / ".config" / "brightdata-cli" + cred_dir.mkdir(parents=True) + (cred_dir / "config.json").write_text(json.dumps({"api_key": "from_config"})) + assert read_cli_credentials() is None + + +# --------------------------------------------------------------------------- +# Client resolution precedence: param -> env -> CLI -> error +# --------------------------------------------------------------------------- + + +class TestResolutionPrecedence: + def test_param_wins_over_env_and_cli(self, monkeypatch): + monkeypatch.setenv("BRIGHTDATA_API_TOKEN", ENV_TOKEN) + monkeypatch.setattr(client_module, "read_cli_credentials", lambda: CLI_TOKEN) + c = BrightDataClient(token=PARAM_TOKEN) + assert c.token == PARAM_TOKEN + assert c.auth_source == "param" + + def test_env_wins_over_cli(self, no_env, monkeypatch): + monkeypatch.setenv("BRIGHTDATA_API_TOKEN", ENV_TOKEN) + monkeypatch.setattr(client_module, "read_cli_credentials", lambda: CLI_TOKEN) + c = BrightDataClient() + assert c.token == ENV_TOKEN + assert c.auth_source == "env" + + def test_alt_env_var(self, no_env, monkeypatch): + monkeypatch.setenv("BRIGHTDATA_API_KEY", ENV_TOKEN) + c = BrightDataClient() + assert c.token == ENV_TOKEN + assert c.auth_source == "env" + + def test_primary_env_var_beats_alt(self, no_env, monkeypatch): + monkeypatch.setenv("BRIGHTDATA_API_TOKEN", ENV_TOKEN) + monkeypatch.setenv("BRIGHTDATA_API_KEY", "other_token_1234567890") + c = BrightDataClient() + assert c.token == ENV_TOKEN + + def test_cli_fallback(self, no_env, monkeypatch): + monkeypatch.setattr(client_module, "read_cli_credentials", lambda: CLI_TOKEN) + c = BrightDataClient() + assert c.token == CLI_TOKEN + assert c.auth_source == "cli_credentials" + + def test_no_credentials_error_is_actionable(self, no_env, monkeypatch): + monkeypatch.setattr(client_module, "read_cli_credentials", lambda: None) + with pytest.raises(ValidationError) as exc_info: + BrightDataClient() + msg = str(exc_info.value) + assert "brightdata login" in msg + assert "BRIGHTDATA_API_TOKEN" in msg + assert "https://brightdata.com/cp/api_keys" in msg + + def test_sync_client_inherits_resolution(self, no_env, monkeypatch): + from brightdata.sync_client import SyncBrightDataClient + + monkeypatch.setattr(client_module, "read_cli_credentials", lambda: CLI_TOKEN) + c = SyncBrightDataClient() + assert c.token == CLI_TOKEN + + +# --------------------------------------------------------------------------- +# User-Agent auth-source reporting +# --------------------------------------------------------------------------- + + +class TestUserAgentAuthSource: + @pytest.mark.parametrize("source", ["param", "env", "cli_credentials"]) + async def test_ua_carries_auth_source(self, source): + engine = AsyncEngine(bearer_token="tok", auth_source=source) + async with engine: + ua = engine._session.headers["User-Agent"] + assert ua.startswith("brightdata-sdk-python/") + assert ua.endswith(f"(auth={source})") + + async def test_ua_without_auth_source(self): + engine = AsyncEngine(bearer_token="tok") + async with engine: + ua = engine._session.headers["User-Agent"] + assert ua.startswith("brightdata-sdk-python/") + assert "(auth=" not in ua + + async def test_token_never_in_user_agent(self): + engine = AsyncEngine(bearer_token="super_secret_token_value", auth_source="env") + async with engine: + assert "super_secret_token_value" not in engine._session.headers["User-Agent"] + + def test_client_wires_auth_source_into_engine(self): + c = BrightDataClient(token=PARAM_TOKEN) + assert c.engine._auth_source == "param" + assert c.auth_source == "param" diff --git a/tests/unit/test_engine.py b/tests/unit/test_engine.py index 2b82641..79922a6 100644 --- a/tests/unit/test_engine.py +++ b/tests/unit/test_engine.py @@ -10,7 +10,6 @@ from brightdata.core.engine import AsyncEngine from brightdata.exceptions import AuthenticationError, NetworkError, SSLError - # --------------------------------------------------------------------------- # Initialization # --------------------------------------------------------------------------- @@ -81,7 +80,9 @@ async def test_session_headers_contain_auth(self): headers = engine._session.headers assert headers["Authorization"] == "Bearer my_secret_token" assert headers["Content-Type"] == "application/json" - assert "brightdata-sdk/" in headers["User-Agent"] + assert "brightdata-sdk-python/" in headers["User-Agent"] + # No auth_source given -> no auth field in the UA + assert "(auth=" not in headers["User-Agent"] # --------------------------------------------------------------------------- From 3ac1b67f5eda5d9a73bd41ecbf9ad57f20ea390d Mon Sep 17 00:00:00 2001 From: "user.mail" Date: Tue, 28 Jul 2026 16:44:18 +0300 Subject: [PATCH 3/3] bump version to 2.5.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bd2706d..0583e5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ where = ["src"] [project] name = "brightdata-sdk" -version = "2.4.1" +version = "2.5.0" description = "Modern async-first Python SDK for Bright Data APIs" authors = [{name = "Bright Data", email = "support@brightdata.com"}] license = {text = "MIT"}