Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 63 additions & 44 deletions python/packages/core/agent_framework/_workflows/_functional.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@
import logging
import math
import typing
from collections.abc import AsyncIterable, Awaitable, Callable, Iterable, Mapping, Sequence
from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Callable, Iterable, Mapping, Sequence
from contextlib import aclosing
from contextvars import Context, ContextVar
from copy import deepcopy
from types import CodeType
Expand Down Expand Up @@ -2061,17 +2062,44 @@ def run(
)
self._ensure_not_running()

released = False

def release_run_guard() -> None:
nonlocal released
if not released:
released = True
self._is_running = False

async def run_guarded() -> AsyncGenerator[WorkflowEvent[Any], None]:
try:
async with aclosing(
self._run_core(
message=message,
responses=responses,
checkpoint_id=checkpoint_id,
checkpoint_storage=checkpoint_storage,
streaming=stream,
**kwargs,
)
) as events:
async for event in events:
yield event
finally:
# Cover initialization failures and abandoned generators as well.
release_run_guard()

guarded_stream = run_guarded()

async def cleanup() -> None:
# An active pull rejects aclose(); retain its guard until it settles.
await guarded_stream.aclose()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After this close races an active pull, ResponseStream.close() marks the stream consumed and consumes this one-shot cleanup hook. Once the pull yields, get_final_response() then finalizes the partial updates without draining or closing guarded_stream, so it returns successfully while the workflow remains locked and every later run() is rejected; this occurs through both direct and as_agent() streams. Please keep failed close recoverable through normal finalization so get_final_response() also releases the guard.

# Closing a never-started generator does not execute its finally block.
release_run_guard()

response_stream: ResponseStream[WorkflowEvent[Any], WorkflowRunResult] = ResponseStream(
self._run_core(
message=message,
responses=responses,
checkpoint_id=checkpoint_id,
checkpoint_storage=checkpoint_storage,
streaming=stream,
**kwargs,
),
guarded_stream,
finalizer=functools.partial(self._finalize_events, include_status_events=include_status_events),
cleanup_hooks=[self._run_cleanup],
cleanup_hooks=[cleanup],
)

if stream:
Expand Down Expand Up @@ -2133,7 +2161,7 @@ async def _run_core(
checkpoint_storage: CheckpointStorage | None = None,
streaming: bool = False,
**kwargs: Any,
) -> AsyncIterable[WorkflowEvent[Any]]:
) -> AsyncGenerator[WorkflowEvent[Any], None]:
storage = checkpoint_storage or self._checkpoint_storage
dependencies = self._discover_workflow_dependencies(self._func)
step_wrappers, _ = dependencies
Expand Down Expand Up @@ -2330,10 +2358,6 @@ async def _on_step_completed() -> None:
capture_exception(span, exception=exc)
raise
finally:
# ResponseStream cleanup_hooks do not run when the generator is
# closed by GC. Release the run lock here so a follow-up run
# after an abandoned stream is not rejected as concurrent.
self._release_run_guard()
span.end()

async def _execute(self, ctx: RunContext, message: Any) -> Any:
Expand Down Expand Up @@ -2664,12 +2688,6 @@ def _ensure_not_running(self) -> None:
raise RuntimeError("Workflow is already running. Concurrent executions are not allowed.")
self._is_running = True

def _release_run_guard(self) -> None:
self._is_running = False

async def _run_cleanup(self) -> None:
self._release_run_guard()


# ---------------------------------------------------------------------------
# @workflow decorator
Expand Down Expand Up @@ -2917,29 +2935,30 @@ def _run_streaming(
)

async def _generate_updates() -> AsyncIterable[AgentResponseUpdate]:
async for event in workflow_stream:
if event.type == "output":
data = event.data
if isinstance(data, str):
contents: list[Content] = [Content.from_text(text=data)]
elif isinstance(data, Content):
contents = [data]
else:
contents = [Content.from_text(text=str(data))]
yield AgentResponseUpdate(
contents=contents,
role="assistant",
author_name=agent_name,
)
elif event.type == "request_info":
approval = self._request_info_to_approval_request(event)
if approval is None:
continue
yield AgentResponseUpdate(
contents=[approval],
role="assistant",
author_name=agent_name,
)
async with workflow_stream:
async for event in workflow_stream:
if event.type == "output":
data = event.data
if isinstance(data, str):
contents: list[Content] = [Content.from_text(text=data)]
elif isinstance(data, Content):
contents = [data]
else:
contents = [Content.from_text(text=str(data))]
yield AgentResponseUpdate(
contents=contents,
role="assistant",
author_name=agent_name,
)
elif event.type == "request_info":
approval = self._request_info_to_approval_request(event)
if approval is None:
continue
yield AgentResponseUpdate(
contents=[approval],
role="assistant",
author_name=agent_name,
)

return ResponseStream(
_generate_updates(),
Expand Down
132 changes: 132 additions & 0 deletions python/packages/core/tests/workflow/test_functional_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -2649,6 +2649,138 @@ async def wf(x: int) -> int:


class TestConcurrencyGuard:
@pytest.mark.parametrize("as_agent", [False, True])
@pytest.mark.parametrize("cancel", [False, True])
async def test_close_racing_active_pull_preserves_guard(self, as_agent: bool, cancel: bool):
started = asyncio.Event()
release = asyncio.Event()
stopped = asyncio.Event()
calls: list[int] = []

@built_workflow
async def wf(value: int) -> str:
calls.append(value)
if value == 1:
try:
started.set()
await release.wait()
finally:
stopped.set()
return f"result: {value}"

runner = wf.as_agent() if as_agent else wf
stream = runner.run(1, stream=True)

async def pull_output() -> None:
async for event in stream:
if isinstance(event, AgentResponseUpdate) or event.type == "output":
return

pull = asyncio.create_task(pull_output())
try:
await asyncio.wait_for(started.wait(), timeout=5)
with pytest.raises(RuntimeError, match="already running"):
await stream.close()
assert not stopped.is_set()
with pytest.raises(RuntimeError, match="Workflow is already running"):
await runner.run(2)
assert calls == [1]

if cancel:
pull.cancel()
with pytest.raises(asyncio.CancelledError):
await pull
else:
release.set()
await asyncio.wait_for(pull, timeout=5)
assert stopped.is_set()

# Retry close after the pending pull settles, without draining the stream.
await stream.close()
await runner.run(2)
assert calls == [1, 2]

newer_stream = runner.run(3, stream=True)
try:
await stream.close()
with pytest.raises(RuntimeError, match="Workflow is already running"):
await runner.run(4)
finally:
await newer_stream.close()
finally:
release.set()
if not pull.done():
pull.cancel()
await asyncio.gather(pull, return_exceptions=True)
await stream.close()

@pytest.mark.parametrize("as_agent", [False, True])
async def test_closed_unstarted_stream_cannot_execute_or_release_new_run(self, as_agent: bool):
calls: list[int] = []

@built_workflow
async def wf(value: int) -> str:
calls.append(value)
return f"result: {value}"

runner = wf.as_agent() if as_agent else wf
old_stream = runner.run(1, stream=True)
await old_stream.close()
newer_stream = runner.run(2, stream=True)
try:
assert [event async for event in old_stream] == []
assert calls == []
await old_stream.close()
with pytest.raises(RuntimeError, match="Workflow is already running"):
await runner.run(3)
await newer_stream.get_final_response()
assert calls == [2]
finally:
await old_stream.close()
await newer_stream.close()

@pytest.mark.parametrize("as_agent", [False, True])
@pytest.mark.parametrize("cancel", [False, True])
async def test_close_racing_checkpoint_load_releases_guard_after_failure(self, as_agent: bool, cancel: bool):
started = asyncio.Event()
release = asyncio.Event()

class WaitingStorage(InMemoryCheckpointStorage):
async def load(self, checkpoint_id: CheckpointID) -> WorkflowCheckpoint:
started.set()
await release.wait()
raise ValueError("checkpoint load failed")

@built_workflow
async def wf(value: int) -> str:
return f"result: {value}"

runner = wf.as_agent() if as_agent else wf
stream = runner.run(checkpoint_id="missing", checkpoint_storage=WaitingStorage(), stream=True)
pull = asyncio.create_task(anext(stream))
try:
await asyncio.wait_for(started.wait(), timeout=5)
with pytest.raises(RuntimeError, match="already running"):
await stream.close()
with pytest.raises(RuntimeError, match="Workflow is already running"):
await runner.run(2)
if cancel:
pull.cancel()
with pytest.raises(asyncio.CancelledError):
await pull
else:
release.set()
with pytest.raises(ValueError, match="checkpoint load failed"):
await asyncio.wait_for(pull, timeout=5)
await stream.close()
await runner.run(2)
finally:
release.set()
if not pull.done():
pull.cancel()
await asyncio.gather(pull, return_exceptions=True)
await stream.close()

async def test_concurrent_run_raises(self):
@built_workflow
async def slow_wf(x: int) -> int:
Expand Down
Loading