Fix: O(n²) parent lookup in BFS and duplicate URL enqueue in BestFirst

crawl4ai/deep_crawling/bff_strategy.py

@@ -216,6 +216,8 @@ async def _arun_best_first(
             queue_items = self._resume_state.get("queue_items", [])
             for item in queue_items:
                 await queue.put((item["score"], item["depth"], item["url"], item["parent_url"]))
+            # `queued` tracks every URL ever pushed onto the queue
+            queued: Set[str] = set(visited) | {item["url"] for item in queue_items}
             # Initialize shadow list if callback is set
             if self._on_state_change:
                 self._queue_shadow = [
@@ -227,6 +229,7 @@ async def _arun_best_first(
             initial_score = self.url_scorer.score(start_url) if self.url_scorer else 0
             await queue.put((-initial_score, 0, start_url, None))
             visited: Set[str] = set()
+            queued: Set[str] = {start_url}
             depths: Dict[str, int] = {start_url: 0}
             # Initialize shadow list if callback is set
             if self._on_state_change:
@@ -313,6 +316,9 @@ async def _arun_best_first(
                     await self.link_discovery(result, url, depth, visited, new_links, depths)
 
                     for new_url, new_parent in new_links:
+                        # Skip URLs already sitting in the queue
+                        if new_url in queued:
+                            continue
                         new_depth = depths.get(new_url, depth + 1)
                         new_score = self.url_scorer.score(new_url) if self.url_scorer else 0
                         # Skip URLs with scores below the threshold
@@ -322,6 +328,7 @@ async def _arun_best_first(
                             )
                             self.stats.urls_skipped += 1
                             continue
+                        queued.add(new_url)
                         queue_item = (-new_score, new_depth, new_url, new_parent)
                         await queue.put(queue_item)
                         # Add to shadow list if tracking

crawl4ai/deep_crawling/bfs_strategy.py

@@ -248,6 +248,7 @@ async def _arun_batch(
 
             next_level: List[Tuple[str, Optional[str]]] = []
             urls = [url for url, _ in current_level]
+            parent_by_url = dict(current_level)
 
             # Clone the config to disable deep crawling recursion and enforce batch mode.
             batch_config = config.clone(deep_crawl_strategy=None, stream=False)
@@ -258,7 +259,7 @@ async def _arun_batch(
                 depth = depths.get(url, 0)
                 result.metadata = result.metadata or {}
                 result.metadata["depth"] = depth
-                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                parent_url = parent_by_url.get(url)
                 result.metadata["parent_url"] = parent_url
                 results.append(result)
 
@@ -336,19 +337,20 @@ async def _arun_stream(
 
             next_level: List[Tuple[str, Optional[str]]] = []
             urls = [url for url, _ in current_level]
+            parent_by_url = dict(current_level)
             visited.update(urls)
 
             stream_config = config.clone(deep_crawl_strategy=None, stream=True)
             stream_gen = await crawler.arun_many(urls=urls, config=stream_config)
-            
+
             # Keep track of processed results for this batch
             results_count = 0
             async for result in stream_gen:
                 url = result.url
                 depth = depths.get(url, 0)
                 result.metadata = result.metadata or {}
                 result.metadata["depth"] = depth
-                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                parent_url = parent_by_url.get(url)
                 result.metadata["parent_url"] = parent_url
 
                 # Count only successful crawls