From d53a2ce15bf0c7f77a2932a7836e0af439423862 Mon Sep 17 00:00:00 2001 From: yongthyuan <824149100@qq.com> Date: Wed, 11 Nov 2026 17:55:23 +0800 Subject: [PATCH] fix: retain functional workflow guard until stream termination --- .../agent_framework/_workflows/_functional.py | 107 ++++++++------ .../workflow/test_functional_workflow.py | 132 ++++++++++++++++++ 2 files changed, 195 insertions(+), 44 deletions(-) diff --git a/python/packages/core/agent_framework/_workflows/_functional.py b/python/packages/core/agent_framework/_workflows/_functional.py index 5bf49d7c76c..36e27d5e0df 100644 --- a/python/packages/core/agent_framework/_workflows/_functional.py +++ b/python/packages/core/agent_framework/_workflows/_functional.py @@ -48,7 +48,8 @@ import logging import math import typing -from collections.abc import AsyncIterable, Awaitable, Callable, Iterable, Mapping, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Callable, Iterable, Mapping, Sequence +from contextlib import aclosing from contextvars import Context, ContextVar from copy import deepcopy from types import CodeType @@ -2061,17 +2062,44 @@ def run( ) self._ensure_not_running() + released = False + + def release_run_guard() -> None: + nonlocal released + if not released: + released = True + self._is_running = False + + async def run_guarded() -> AsyncGenerator[WorkflowEvent[Any], None]: + try: + async with aclosing( + self._run_core( + message=message, + responses=responses, + checkpoint_id=checkpoint_id, + checkpoint_storage=checkpoint_storage, + streaming=stream, + **kwargs, + ) + ) as events: + async for event in events: + yield event + finally: + # Cover initialization failures and abandoned generators as well. + release_run_guard() + + guarded_stream = run_guarded() + + async def cleanup() -> None: + # An active pull rejects aclose(); retain its guard until it settles. + await guarded_stream.aclose() + # Closing a never-started generator does not execute its finally block. + release_run_guard() + response_stream: ResponseStream[WorkflowEvent[Any], WorkflowRunResult] = ResponseStream( - self._run_core( - message=message, - responses=responses, - checkpoint_id=checkpoint_id, - checkpoint_storage=checkpoint_storage, - streaming=stream, - **kwargs, - ), + guarded_stream, finalizer=functools.partial(self._finalize_events, include_status_events=include_status_events), - cleanup_hooks=[self._run_cleanup], + cleanup_hooks=[cleanup], ) if stream: @@ -2133,7 +2161,7 @@ async def _run_core( checkpoint_storage: CheckpointStorage | None = None, streaming: bool = False, **kwargs: Any, - ) -> AsyncIterable[WorkflowEvent[Any]]: + ) -> AsyncGenerator[WorkflowEvent[Any], None]: storage = checkpoint_storage or self._checkpoint_storage dependencies = self._discover_workflow_dependencies(self._func) step_wrappers, _ = dependencies @@ -2330,10 +2358,6 @@ async def _on_step_completed() -> None: capture_exception(span, exception=exc) raise finally: - # ResponseStream cleanup_hooks do not run when the generator is - # closed by GC. Release the run lock here so a follow-up run - # after an abandoned stream is not rejected as concurrent. - self._release_run_guard() span.end() async def _execute(self, ctx: RunContext, message: Any) -> Any: @@ -2664,12 +2688,6 @@ def _ensure_not_running(self) -> None: raise RuntimeError("Workflow is already running. Concurrent executions are not allowed.") self._is_running = True - def _release_run_guard(self) -> None: - self._is_running = False - - async def _run_cleanup(self) -> None: - self._release_run_guard() - # --------------------------------------------------------------------------- # @workflow decorator @@ -2917,29 +2935,30 @@ def _run_streaming( ) async def _generate_updates() -> AsyncIterable[AgentResponseUpdate]: - async for event in workflow_stream: - if event.type == "output": - data = event.data - if isinstance(data, str): - contents: list[Content] = [Content.from_text(text=data)] - elif isinstance(data, Content): - contents = [data] - else: - contents = [Content.from_text(text=str(data))] - yield AgentResponseUpdate( - contents=contents, - role="assistant", - author_name=agent_name, - ) - elif event.type == "request_info": - approval = self._request_info_to_approval_request(event) - if approval is None: - continue - yield AgentResponseUpdate( - contents=[approval], - role="assistant", - author_name=agent_name, - ) + async with workflow_stream: + async for event in workflow_stream: + if event.type == "output": + data = event.data + if isinstance(data, str): + contents: list[Content] = [Content.from_text(text=data)] + elif isinstance(data, Content): + contents = [data] + else: + contents = [Content.from_text(text=str(data))] + yield AgentResponseUpdate( + contents=contents, + role="assistant", + author_name=agent_name, + ) + elif event.type == "request_info": + approval = self._request_info_to_approval_request(event) + if approval is None: + continue + yield AgentResponseUpdate( + contents=[approval], + role="assistant", + author_name=agent_name, + ) return ResponseStream( _generate_updates(), diff --git a/python/packages/core/tests/workflow/test_functional_workflow.py b/python/packages/core/tests/workflow/test_functional_workflow.py index 2c3271fbf4f..ba7f25cfd93 100644 --- a/python/packages/core/tests/workflow/test_functional_workflow.py +++ b/python/packages/core/tests/workflow/test_functional_workflow.py @@ -2649,6 +2649,138 @@ async def wf(x: int) -> int: class TestConcurrencyGuard: + @pytest.mark.parametrize("as_agent", [False, True]) + @pytest.mark.parametrize("cancel", [False, True]) + async def test_close_racing_active_pull_preserves_guard(self, as_agent: bool, cancel: bool): + started = asyncio.Event() + release = asyncio.Event() + stopped = asyncio.Event() + calls: list[int] = [] + + @built_workflow + async def wf(value: int) -> str: + calls.append(value) + if value == 1: + try: + started.set() + await release.wait() + finally: + stopped.set() + return f"result: {value}" + + runner = wf.as_agent() if as_agent else wf + stream = runner.run(1, stream=True) + + async def pull_output() -> None: + async for event in stream: + if isinstance(event, AgentResponseUpdate) or event.type == "output": + return + + pull = asyncio.create_task(pull_output()) + try: + await asyncio.wait_for(started.wait(), timeout=5) + with pytest.raises(RuntimeError, match="already running"): + await stream.close() + assert not stopped.is_set() + with pytest.raises(RuntimeError, match="Workflow is already running"): + await runner.run(2) + assert calls == [1] + + if cancel: + pull.cancel() + with pytest.raises(asyncio.CancelledError): + await pull + else: + release.set() + await asyncio.wait_for(pull, timeout=5) + assert stopped.is_set() + + # Retry close after the pending pull settles, without draining the stream. + await stream.close() + await runner.run(2) + assert calls == [1, 2] + + newer_stream = runner.run(3, stream=True) + try: + await stream.close() + with pytest.raises(RuntimeError, match="Workflow is already running"): + await runner.run(4) + finally: + await newer_stream.close() + finally: + release.set() + if not pull.done(): + pull.cancel() + await asyncio.gather(pull, return_exceptions=True) + await stream.close() + + @pytest.mark.parametrize("as_agent", [False, True]) + async def test_closed_unstarted_stream_cannot_execute_or_release_new_run(self, as_agent: bool): + calls: list[int] = [] + + @built_workflow + async def wf(value: int) -> str: + calls.append(value) + return f"result: {value}" + + runner = wf.as_agent() if as_agent else wf + old_stream = runner.run(1, stream=True) + await old_stream.close() + newer_stream = runner.run(2, stream=True) + try: + assert [event async for event in old_stream] == [] + assert calls == [] + await old_stream.close() + with pytest.raises(RuntimeError, match="Workflow is already running"): + await runner.run(3) + await newer_stream.get_final_response() + assert calls == [2] + finally: + await old_stream.close() + await newer_stream.close() + + @pytest.mark.parametrize("as_agent", [False, True]) + @pytest.mark.parametrize("cancel", [False, True]) + async def test_close_racing_checkpoint_load_releases_guard_after_failure(self, as_agent: bool, cancel: bool): + started = asyncio.Event() + release = asyncio.Event() + + class WaitingStorage(InMemoryCheckpointStorage): + async def load(self, checkpoint_id: CheckpointID) -> WorkflowCheckpoint: + started.set() + await release.wait() + raise ValueError("checkpoint load failed") + + @built_workflow + async def wf(value: int) -> str: + return f"result: {value}" + + runner = wf.as_agent() if as_agent else wf + stream = runner.run(checkpoint_id="missing", checkpoint_storage=WaitingStorage(), stream=True) + pull = asyncio.create_task(anext(stream)) + try: + await asyncio.wait_for(started.wait(), timeout=5) + with pytest.raises(RuntimeError, match="already running"): + await stream.close() + with pytest.raises(RuntimeError, match="Workflow is already running"): + await runner.run(2) + if cancel: + pull.cancel() + with pytest.raises(asyncio.CancelledError): + await pull + else: + release.set() + with pytest.raises(ValueError, match="checkpoint load failed"): + await asyncio.wait_for(pull, timeout=5) + await stream.close() + await runner.run(2) + finally: + release.set() + if not pull.done(): + pull.cancel() + await asyncio.gather(pull, return_exceptions=True) + await stream.close() + async def test_concurrent_run_raises(self): @built_workflow async def slow_wf(x: int) -> int: