Skip to content

[Bug]: Deep crawling crashes from time to time #1917

@Qbason

Description

@Qbason

crawl4ai version

0.8.5

Expected Behavior

Current Behavior

[2026-04-13 06:23:51 +0000] [15] [ERROR] Exception in ASGI application

  • Exception Group Traceback (most recent call last):
    | File "/usr/local/lib/python3.12/site-packages/starlette/_utils.py", line 81, in collapse_excgroups
    | yield
    | File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 270, in call
    | async with anyio.create_task_group() as task_group:
    | ^^^^^^^^^^^^^^^^^^^^^^^^^
    | File "/usr/local/lib/python3.12/site-packages/anyio/_backends/_asyncio.py", line 772, in aexit
    | raise BaseExceptionGroup(
    | ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
    +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    | File "/usr/local/lib/python3.12/site-packages/uvicorn/protocols/http/h11_impl.py", line 410, in run_asgi
    | result = await app( # type: ignore[func-returns-value]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    | File "/usr/local/lib/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in call
    | return await self.app(scope, receive, send)
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    | File "/usr/local/lib/python3.12/site-packages/fastapi/applications.py", line 1160, in call
    | await super().call(scope, receive, send)
    | File "/usr/local/lib/python3.12/site-packages/starlette/applications.py", line 107, in call
    | await self.middleware_stack(scope, receive, send)
    | File "/usr/local/lib/python3.12/site-packages/starlette/middleware/errors.py", line 186, in call
    | raise exc
    | File "/usr/local/lib/python3.12/site-packages/starlette/middleware/errors.py", line 164, in call
    | await self.app(scope, receive, _send)
    | File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 198, in call
    | raise app_exc
    | File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 144, in coro
    | await self.app(scope, receive_or_disconnect, send_no_error)
    | File "/usr/local/lib/python3.12/site-packages/prometheus_fastapi_instrumentator/middleware.py", line 177, in call
    | raise exc
    | File "/usr/local/lib/python3.12/site-packages/prometheus_fastapi_instrumentator/middleware.py", line 175, in call
    | await self.app(scope, receive, send_wrapper)
    | File "/usr/local/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 63, in call
    | await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
    | File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    | raise exc
    | File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
    | await app(scope, receive, sender)
    | File "/usr/local/lib/python3.12/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in call
    | await self.app(scope, receive, send)
    | File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 716, in call
    | await self.middleware_stack(scope, receive, send)
    | File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 736, in app
    | await route.handle(scope, receive, send)
    | File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 290, in handle
    | await self.app(scope, receive, send)
    | File "/usr/local/lib/python3.12/site-packages/fastapi/routing.py", line 130, in app
    | await wrap_app_handling_exceptions(app, request)(scope, receive, send)
    | File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    | raise exc
    | File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
    | await app(scope, receive, sender)
    | File "/usr/local/lib/python3.12/site-packages/fastapi/routing.py", line 117, in app
    | await response(scope, receive, send)
    | File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 269, in call
    | with collapse_excgroups():
    | ^^^^^^^^^^^^^^^^^^^^
    | File "/usr/local/lib/python3.12/contextlib.py", line 158, in exit
    | self.gen.throw(value)
    | File "/usr/local/lib/python3.12/site-packages/starlette/_utils.py", line 87, in collapse_excgroups
    | raise exc
    | File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 273, in wrap
    | await func()
    | File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 253, in stream_response
    | async for chunk in self.body_iterator:
    | File "/app/api.py", line 513, in stream_results
    | async for result in results_gen:
    | File "/usr/local/lib/python3.12/site-packages/crawl4ai/deep_crawling/base_strategy.py", line 35, in result_wrapper
    | self.deep_crawl_active.reset(token)
    | ValueError: <Token var=<ContextVar name='deep_crawl_active' default=False at 0x7b6e7c0a2c00> at 0x7b6e78d2bb00> was created in a different Context
    +------------------------------------

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/lib/python3.12/site-packages/uvicorn/protocols/http/h11_impl.py", line 410, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in call
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/fastapi/applications.py", line 1160, in call
await super().call(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/applications.py", line 107, in call
await self.middleware_stack(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/errors.py", line 186, in call
raise exc
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/errors.py", line 164, in call
await self.app(scope, receive, _send)
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 198, in call
raise app_exc
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/base.py", line 144, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/usr/local/lib/python3.12/site-packages/prometheus_fastapi_instrumentator/middleware.py", line 177, in call
raise exc
File "/usr/local/lib/python3.12/site-packages/prometheus_fastapi_instrumentator/middleware.py", line 175, in call
await self.app(scope, receive, send_wrapper)
File "/usr/local/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 63, in call
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/usr/local/lib/python3.12/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in call
await self.app(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 716, in call
await self.middleware_stack(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 736, in app
await route.handle(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/routing.py", line 290, in handle
await self.app(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/fastapi/routing.py", line 130, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/usr/local/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/usr/local/lib/python3.12/site-packages/fastapi/routing.py", line 117, in app
await response(scope, receive, send)
File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 269, in call
with collapse_excgroups():
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/contextlib.py", line 158, in exit
self.gen.throw(value)
File "/usr/local/lib/python3.12/site-packages/starlette/_utils.py", line 87, in collapse_excgroups
raise exc
File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 273, in wrap
await func()
File "/usr/local/lib/python3.12/site-packages/starlette/responses.py", line 253, in stream_response
async for chunk in self.body_iterator:
File "/app/api.py", line 513, in stream_results
async for result in results_gen:
File "/usr/local/lib/python3.12/site-packages/crawl4ai/deep_crawling/base_strategy.py", line 35, in result_wrapper
self.deep_crawl_active.reset(token)
ValueError: <Token var=<ContextVar name='deep_crawl_active' default=False at 0x7b6e7c0a2c00> at 0x7b6e78d2bb00> was created in a different Context

Is this reproducible?

Yes

Inputs Causing the Bug

I do deep crawling and this happens from time to time.
I use docker image.

Steps to Reproduce

Code snippets

{
			Urls = urls,
			BrowserConfig = new BrowserConfig
			{
				Type = "BrowserConfig",
				Params = new BrowserConfigParams
				{
					Headless = true,
					EnableStealth = false,
					MemorySavingMode = true,
					MaxPagesBeforeRecycle = 100,
					AvoidAds = true,
					AvoidCss = true,
				}
			},
			CrawlerConfig = new CrawlerRunConfig
			{
				Type = "CrawlerRunConfig",
				Params = new CrawlerRunConfigParams
				{
					RemoveConsentPopups = true,
					Stream = false,
					CacheMode = "ENABLED",
					WaitUntil = config.WaitUntil switch
					{
						WaitUntilOption.NetworkIdle => "networkidle",
						WaitUntilOption.DomContentLoaded => "domcontentloaded",
						_ => "networkidle"
					},
					Magic = false,
					SimulateUser = false,
					OverrideNavigator = false,
					RemoveOverlayElements = true,
					DeepCrawlStrategy = new DeepCrawlStrategy
      		                       {
      			                 Type = "BFSDeepCrawlStrategy",
      			                 Params = new DeepCrawlStrategyParams
      			                 {
      				            MaxDepth = config.MaxDepth,
      				            MaxPages = config.MaxPages,
      				            FilterChain = BuildFilterChain(config)
      			                 }
      		                       },
					MarkdownGenerator = new MarkdownGenerator
					{
						Type = "DefaultMarkdownGenerator",
						Params = new MarkdownGeneratorParams
						{
							ContentSource = "fit_html",
							Options = new MarkdownGeneratorOptions
							{
								Type = "dict",
								Value = new MarkdownGeneratorOptionValues
								{
									IgnoreLinks = true,
									EscapeHtml = false,
									BodyWidth = 80,
								}
							},
							ContentFilter = new ContentFilter
							{
								Type = "PruningContentFilter",
								Params = new ContentFilterParams
								{
									Threshold = 0.48,
									ThresholdType = "fixed",
									MinWordThreshold = 0
								}
							}
						}
					}
				}
			}
		};

OS

Linux

Python version

Browser

No response

Browser version

No response

Error logs & Screenshots (if applicable)

No response

Metadata

Metadata

Assignees

No one assigned

    Labels

    🐞 BugSomething isn't working🩺 Needs TriageNeeds attention of maintainers

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions