From 97797bde74bc95b3924e56578fc1dcf5f52292a0 Mon Sep 17 00:00:00 2001
From: Solaris-star <820622658@qq.com>
Date: Mon, 20 Jul 2026 18:56:42 +0800
Subject: [PATCH] fix(crawler): await dispatcher stream aclose in arun_many

Hold an explicit reference to run_urls_stream and always aclose() it in
the public generator's finally block so dispatcher-owned crawl tasks are
cancelled and awaited before proxy-session release.

Fixes #2083.
---
 crawl4ai/async_webcrawler.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py
index 8216d19bc..d102fcac2 100644
--- a/crawl4ai/async_webcrawler.py
+++ b/crawl4ai/async_webcrawler.py
@@ -1107,13 +1107,19 @@ async def maybe_release_session():
 
         if stream:
             async def result_transformer():
+                # Hold an explicit reference so aclose() on the public generator
+                # always awaits dispatcher-owned task cleanup (#2083). Relying
+                # only on async-for teardown can release the proxy session while
+                # crawl tasks from the closed stream are still running.
+                stream_gen = dispatcher.run_urls_stream(
+                    crawler=self, urls=urls, config=config
+                )
                 try:
-                    async for task_result in dispatcher.run_urls_stream(
-                        crawler=self, urls=urls, config=config
-                    ):
+                    async for task_result in stream_gen:
                         yield transform_result(task_result)
                 finally:
-                    # Auto-release session after streaming completes
+                    await stream_gen.aclose()
+                    # Auto-release session after dispatcher cleanup finishes
                     await maybe_release_session()
 
             return result_transformer()