Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions python/packages/core/agent_framework/_workflows/_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,10 @@ async def run_until_convergence(self) -> AsyncGenerator[WorkflowEvent, None]:
yield WorkflowEvent.superstep_started(iteration=self._iteration + 1)

committed = False
superstep_epoch = self._state.begin_epoch()
try:
# Wake on either a live event or iteration completion, including silent supersteps.
iteration_task = asyncio.create_task(self._run_iteration())
iteration_task = self._state.bound_context().run(asyncio.create_task, self._run_iteration())
event_task: asyncio.Task[WorkflowEvent] | None = None
try:
while not iteration_task.done():
Expand Down Expand Up @@ -167,7 +168,8 @@ async def run_until_convergence(self) -> AsyncGenerator[WorkflowEvent, None]:
self._state.commit()
committed = True
finally:
if not committed:
# A stale run's finalizer must not discard pending writes owned by a newer superstep.
if not committed and self._state.epoch is superstep_epoch:
self._state.discard()

# Create checkpoint after each superstep iteration
Expand Down Expand Up @@ -243,6 +245,7 @@ async def _prepare_checkpoint_state(self) -> None:
This is used by checkpoint capture paths that need a complete, restorable
state payload without necessarily writing to a checkpoint storage backend.
"""
self._state.begin_epoch()
await self._save_executor_states()
self._save_edge_runner_states()
self._state.commit()
Expand Down
49 changes: 49 additions & 0 deletions python/packages/core/agent_framework/_workflows/_state.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,17 @@
# Copyright (c) Microsoft. All rights reserved.

import copy
import logging
from collections.abc import Callable
from contextvars import Context, ContextVar, copy_context
from typing import Any

logger = logging.getLogger(__name__)

# The State and epoch that writes from the current task belong to. Set for each superstep's iteration
# task so executor tasks inherit it; writes made outside any superstep are always accepted.
_bound_epoch: ContextVar[tuple["State", object] | None] = ContextVar("_bound_epoch", default=None)


class State:
"""Manages shared state across executors within a workflow.
Expand All @@ -18,6 +26,9 @@ class State:
- `get()` checks pending buffer first, then committed state
- `commit()` moves all pending changes to committed state (called by Runner at superstep boundary)
- `discard()` clears pending changes without committing
- `begin_epoch()` starts a new owner of the state; `set()`, `delete()`, `clear()`, `commit()`,
`discard()`, `import_state()` and `begin_epoch()` from tasks bound to an earlier epoch (for
example a dropped run's cancelled executor) are ignored

Reserved Keys:
Keys starting with underscore (_) are reserved for internal framework use.
Expand All @@ -28,6 +39,32 @@ def __init__(self) -> None:
"""Initialize the state."""
self._committed: dict[str, Any] = {}
self._pending: dict[str, Any] = {}
self._epoch = object()

@property
def epoch(self) -> object:
"""The current write epoch."""
return self._epoch

def begin_epoch(self) -> object:
"""Start a new write epoch and return it."""
if self._is_stale_write():
return self._epoch
self._epoch = object()
return self._epoch

def bound_context(self) -> Context:
"""Return a copy of the current context whose writes, and its tasks' writes, belong to the current epoch."""
context = copy_context()
context.run(_bound_epoch.set, (self, self._epoch))
return context

def _is_stale_write(self) -> bool:
bound = _bound_epoch.get()
if bound is None or bound[0] is not self or bound[1] is self._epoch:
return False
logger.debug("Ignoring a state write from a superstep that no longer owns this state.")
return True

def set(self, key: str, value: Any) -> None:
"""Set a value in the pending state buffer.
Expand All @@ -43,6 +80,8 @@ def set(self, key: str, value: Any) -> None:
the .NET behavior and the superstep execution model where all executors
in a superstep see the same committed state at the start.
"""
if self._is_stale_write():
return
self._pending[key] = copy.deepcopy(value)

def get(self, key: str, default: Any = None) -> Any:
Expand Down Expand Up @@ -88,6 +127,8 @@ def delete(self, key: str) -> None:
to indicate deletion at commit time. If it only exists in pending,
it is removed from pending.
"""
if self._is_stale_write():
return
if key not in self._pending and key not in self._committed:
raise KeyError(f"Key '{key}' not found in state.")

Expand All @@ -100,6 +141,8 @@ def delete(self, key: str) -> None:

def clear(self) -> None:
"""Clear both committed and pending state."""
if self._is_stale_write():

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

digit50 Stale contexts can still mutate successor state through other State methods — WorkflowContext.state exposes the shared State object, but epoch checks are only applied to set, delete, and clear. A late cancellation handler from a dropped run can call ctx.state.discard() after the successor has staged writes, clearing those writes; commit() can publish them before the superstep boundary, and import_state() can directly alter committed state. These mutations bypass the ownership guard and let a stale run affect its successor.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now fixed in 9af013f: commit, discard and import_state get the same stale-epoch check as set, delete and clear, and so does begin_epoch, which a stale handler could otherwise call to make the successor's own writes look stale. Framework callers of these methods all run outside any superstep's bound context, so they are unaffected. Regression: test_workflow_partial_stream_cancellation_state_operation_ignored_for_successor, parametrized over the four methods.

return
self._committed.clear()
self._pending.clear()

Expand All @@ -108,6 +151,8 @@ def commit(self) -> None:

Called by the Runner at superstep boundaries after successful execution.
"""
if self._is_stale_write():
return
for key, value in self._pending.items():
if value is _DeleteSentinel:
self._committed.pop(key, None)
Expand All @@ -117,6 +162,8 @@ def commit(self) -> None:

def discard(self) -> None:
"""Discard all pending changes without committing."""
if self._is_stale_write():
return
self._pending.clear()

def export_state(self) -> dict[str, Any]:
Expand All @@ -133,6 +180,8 @@ def import_state(self, state: dict[str, Any]) -> None:
Merges a deepcopy into committed state. Does not affect pending changes.
Values must support :func:`copy.deepcopy`.
"""
if self._is_stale_write():
return
self._committed.update(copy.deepcopy(state))


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -589,6 +589,11 @@ async def _run_workflow_with_tracing(
if not is_continuation:
self._runner.reset_iteration_count()

# Pending writes here come from a superstep or checkpoint save that never committed; do not
# commit them, and ignore later writes from that superstep's executors if it is still finalizing.
self._runner.state.begin_epoch()
self._runner.state.discard()

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This discard only removes predecessor writes already pending when B reaches this point. A's finalizer can run later: its cancelled handler may call ctx.set_state() from finally after this discard, and once B has replaced _superstep_token, A's cleanup skips its own discard. B's next superstep commit can then persist A's cleanup write. Please drain or track predecessor finalization, or isolate pending state by run/epoch, and add a regression where A writes state during cancellation after B has started.

@digit50 digit50 Oct 22, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, good catch. Fixed in e92c3f9: pending writes are now scoped per superstep, so the dropped run's cleanup write is ignored. New test: test_workflow_partial_stream_cancellation_write_not_committed_by_successor.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, good catch. Fixed in e92c3f9: pending writes are now scoped per superstep, so the dropped run's cleanup write is ignored. Regression: test_workflow_partial_stream_cancellation_write_not_committed_by_successor.


# Store run kwargs in State so executors can access them.
# Per-run kwargs semantics:
# - On a fresh message run, prior kwargs go away (set to {}
Expand Down
Loading
Loading