diff --git a/python/packages/core/agent_framework/_harness/_todo.py b/python/packages/core/agent_framework/_harness/_todo.py index 7aaeeecb2a3..722c0c5b123 100644 --- a/python/packages/core/agent_framework/_harness/_todo.py +++ b/python/packages/core/agent_framework/_harness/_todo.py @@ -244,6 +244,18 @@ async def load_items(self, session: AgentSession, *, source_id: str) -> list[Tod items, _ = await self.load_state(session, source_id=source_id) return items + def _mutation_identity(self, session: AgentSession, *, source_id: str) -> object: + """Return the hashable identity that mutations on ``session`` must serialize on. + + In-memory stores keep their state on the session object itself, so the session + instance identifies the storage. Stores that map several session objects, or + several store instances, onto one backing file override this to return a stable + identity of that file instead, so concurrent mutations share one lock. Override + values carry their own namespace tag, so identities of differently backed stores + never collide. + """ + return ("session", id(session)) + class TodoSessionStore(TodoStore): """Store todo state inside ``AgentSession.state``.""" @@ -362,6 +374,15 @@ def _is_literal_path_segment_safe(cls, value: str) -> bool: """Return whether a value can be used directly as one path segment.""" return _is_literal_storage_key_segment_safe(value) + def _mutation_identity(self, session: AgentSession, *, source_id: str) -> object: + """Serialize mutations on the resolved state file rather than on session objects. + + Two sessions with the same session ID (and owner scope), or two store instances + over one directory, resolve to the same JSON file; locking per session object + would let them load the same state concurrently and overwrite each other. + """ + return ("file", str(self._get_state_path(session, source_id=source_id))) + def _state_filename(self, source_id: str) -> str: """Return a source-specific JSON state filename.""" state_path = Path(self.state_filename) @@ -411,6 +432,62 @@ def _save_state_sync(state_path: Path, payload: str) -> None: _atomic_write_text(state_path, payload) +# Locks handed out by ``_todo_mutation_lock``. Values are held weakly, so an entry lasts only while a +# mutation holds or awaits its lock, and the map never grows with the number of sessions ever seen. +_TODO_MUTATION_LOCKS: weakref.WeakValueDictionary[tuple[asyncio.AbstractEventLoop, object], asyncio.Lock] = ( + weakref.WeakValueDictionary() +) + + +def _todo_mutation_lock(store: TodoStore, session: AgentSession, *, source_id: str) -> asyncio.Lock: + """Return the lock that serializes read-modify-write todo operations on one backing storage. + + The lock key is the store's ``_mutation_identity`` for the session, so two providers + sharing a store — and, for stores backed by files, two sessions or two store + instances that resolve to the same backing file — cannot both load the state and + ``next_id`` before either writes them back and silently drop one of the updates. + Unrelated identities keep separate locks, so unrelated sessions still mutate + concurrently. + + Locks are per event loop, because an :class:`asyncio.Lock` that has had a waiter on + one loop raises when a caller has to wait for it on another, and one session can be + reused across ``asyncio.run`` boundaries. + + The lock is process-local: other processes, and storage mappings that bypass + ``_mutation_identity``, are not coordinated. + """ + key = (asyncio.get_running_loop(), store._mutation_identity(session, source_id=source_id)) + lock = _TODO_MUTATION_LOCKS.get(key) + if lock is None: + lock = asyncio.Lock() + _TODO_MUTATION_LOCKS[key] = lock + return lock + + +async def _save_state_holding_lock( + store: TodoStore, session: AgentSession, items: list[TodoItem], *, next_id: int, source_id: str +) -> None: + """Persist todo state while keeping the mutation lock until the write has finished. + + ``TodoFileStore`` saves through a worker thread. If the caller is cancelled while + that write is in flight, releasing the lock straight away would let the next + mutation save first and then be overwritten by the stale in-flight write. The save + is awaited to completion here, and the cancellation is re-raised afterwards. + """ + save_task = asyncio.ensure_future(store.save_state(session, items, next_id=next_id, source_id=source_id)) + try: + await asyncio.shield(save_task) + except asyncio.CancelledError: + while not save_task.done(): + try: + await asyncio.shield(save_task) + except asyncio.CancelledError: + pass # cancelled again while waiting; the in-flight write still comes first + if not save_task.cancelled(): + save_task.exception() # retrieved so the finished task does not warn + raise + + class TodoProvider(ContextProvider): """Provide todo management tools and instructions to an agent. @@ -446,17 +523,10 @@ def __init__( super().__init__(source_id) self.instructions = instructions or DEFAULT_TODO_INSTRUCTIONS self.store = store or TodoSessionStore() - # WeakKeyDictionary so per-session locks are evicted automatically when the session is GC'd - # rather than accumulating in long-running services that create many sessions. - self._mutation_locks: weakref.WeakKeyDictionary[AgentSession, asyncio.Lock] = weakref.WeakKeyDictionary() def _mutation_lock(self, session: AgentSession) -> asyncio.Lock: - """Return the per-session lock for read-modify-write todo operations.""" - lock = self._mutation_locks.get(session) - if lock is None: - lock = asyncio.Lock() - self._mutation_locks[session] = lock - return lock + """Return the storage-scoped lock for read-modify-write todo operations.""" + return _todo_mutation_lock(self.store, session, source_id=self.source_id) async def before_run( self, @@ -490,7 +560,9 @@ async def todos_add(todos: list[_TodoAddItemSchema]) -> str: created_items.append(created_item) next_id += 1 - await self.store.save_state(session, existing_items, next_id=next_id, source_id=self.source_id) + await _save_state_holding_lock( + self.store, session, existing_items, next_id=next_id, source_id=self.source_id + ) return json.dumps([item.to_dict(exclude_none=False) for item in created_items]) @tool(name="todos_complete", approval_mode="never_require") @@ -525,7 +597,9 @@ async def todos_complete(items: list[_TodoCompleteItemSchema]) -> str: updated_items.append(item) if completed_count: - await self.store.save_state(session, updated_items, next_id=next_id, source_id=self.source_id) + await _save_state_holding_lock( + self.store, session, updated_items, next_id=next_id, source_id=self.source_id + ) return json.dumps({"completed": completed_count}) @tool(name="todos_remove", approval_mode="never_require") @@ -539,7 +613,9 @@ async def todos_remove(ids: list[int]) -> str: remaining_items = [item for item in items if item.id not in set(ids)] removed_count = len(items) - len(remaining_items) if removed_count: - await self.store.save_state(session, remaining_items, next_id=next_id, source_id=self.source_id) + await _save_state_holding_lock( + self.store, session, remaining_items, next_id=next_id, source_id=self.source_id + ) return json.dumps({"removed": removed_count}) @tool(name="todos_get_remaining", approval_mode="never_require") diff --git a/python/packages/core/tests/core/test_harness_todo.py b/python/packages/core/tests/core/test_harness_todo.py index 2b76c1b49ce..52230b98fff 100644 --- a/python/packages/core/tests/core/test_harness_todo.py +++ b/python/packages/core/tests/core/test_harness_todo.py @@ -3,8 +3,10 @@ from __future__ import annotations import asyncio +import contextlib import json import os +import threading from pathlib import Path import pytest @@ -199,18 +201,63 @@ async def test_todo_stores_clamp_next_id_to_avoid_collisions(tmp_path: Path) -> assert items == [TodoItem(id=7, title="Seeded")] -async def test_todo_provider_evicts_locks_when_session_is_garbage_collected() -> None: - """The provider should not retain mutation locks for sessions that have been GC'd.""" +async def test_todo_mutation_locks_are_evicted_when_no_longer_needed() -> None: + """Mutation locks should not be retained once no mutation holds or awaits them.""" import gc - provider = TodoProvider() + from agent_framework._harness._todo import _TODO_MUTATION_LOCKS + + store = TodoSessionStore() + provider = TodoProvider(store=store) session = AgentSession(session_id="session-1") - provider._mutation_lock(session) # pyright: ignore[reportPrivateUsage] - assert len(provider._mutation_locks) == 1 # pyright: ignore[reportPrivateUsage] + lock = provider._mutation_lock(session) # pyright: ignore[reportPrivateUsage] + assert len(_TODO_MUTATION_LOCKS) == 1 - del session + del lock, session, provider gc.collect() - assert len(provider._mutation_locks) == 0 # pyright: ignore[reportPrivateUsage] + assert len(_TODO_MUTATION_LOCKS) == 0 + + +async def test_todo_mutation_lock_is_scoped_to_the_storage_identity() -> None: + """Providers over one storage identity share a lock; another identity gets its own.""" + session = AgentSession(session_id="session-1") + first_provider = TodoProvider(store=TodoSessionStore()) + second_provider = TodoProvider(store=TodoSessionStore()) + other_session_provider = TodoProvider(store=TodoSessionStore()) + + lock = first_provider._mutation_lock(session) # pyright: ignore[reportPrivateUsage] + # Session stores keep their state on the session object itself, so both providers — + # and both stores — serialize on that one session instance. + assert second_provider._mutation_lock(session) is lock # pyright: ignore[reportPrivateUsage] + assert other_session_provider._mutation_lock(AgentSession(session_id="session-1")) is not lock # pyright: ignore[reportPrivateUsage] + + +async def test_todo_mutation_lock_shares_one_file_identity_for_equivalent_sessions(tmp_path: Path) -> None: + """Sessions that resolve to the same todo file must share one mutation lock.""" + provider = TodoProvider(store=TodoFileStore(tmp_path)) + first_lock = provider._mutation_lock(AgentSession(session_id="session-1")) # pyright: ignore[reportPrivateUsage] + second_lock = provider._mutation_lock(AgentSession(session_id="session-1")) # pyright: ignore[reportPrivateUsage] + assert first_lock is second_lock + + # A different session ID resolves to a different file and keeps its own lock. + assert provider._mutation_lock(AgentSession(session_id="session-2")) is not first_lock # pyright: ignore[reportPrivateUsage] + + # Two store instances over the same directory resolve to the same file as well. + other_provider = TodoProvider(store=TodoFileStore(tmp_path)) + assert other_provider._mutation_lock(AgentSession(session_id="session-1")) is first_lock # pyright: ignore[reportPrivateUsage] + + +def test_todo_mutation_lock_is_scoped_to_the_event_loop() -> None: + """Reusing a session across ``asyncio.run`` boundaries must not reuse a loop-bound lock.""" + + async def _lock_from_new_loop(provider: TodoProvider, session: AgentSession) -> asyncio.Lock: + return provider._mutation_lock(session) # pyright: ignore[reportPrivateUsage] + + provider = TodoProvider() + session = AgentSession(session_id="session-1") + first_lock = asyncio.run(_lock_from_new_loop(provider, session)) + second_lock = asyncio.run(_lock_from_new_loop(provider, session)) + assert first_lock is not second_lock async def test_todo_file_store_rejects_session_path_traversal(tmp_path: Path) -> None: @@ -368,6 +415,175 @@ async def test_todo_provider_serializes_concurrent_mutations( assert {item["id"] for item in payload if item["is_complete"]} == {1, 2, 3, 4, 5} +async def test_todo_providers_sharing_one_store_serialize_mutations( + chat_client_base: SupportsChatGetResponse, +) -> None: + """Two providers over one shared store must not load the same state and drop an add.""" + + class _GatedSharedTodoStore(TodoStore): + """Hold each load until a competing caller also finishes loading, so the loads interleave.""" + + GATE_TIMEOUT = 0.25 + + def __init__(self) -> None: + self.items: list[TodoItem] = [] + self.next_id = 1 + self.load_count = 0 + self.both_loaded = asyncio.Event() + + def reset_gate(self) -> None: + self.load_count = 0 + self.both_loaded.clear() + + async def load_state(self, session: AgentSession, *, source_id: str) -> tuple[list[TodoItem], int]: + snapshot = (list(self.items), self.next_id) + self.load_count += 1 + if self.load_count >= 2: + self.both_loaded.set() + # A caller whose mutations are already serialized never sees a competing load, + # so bound the wait instead of blocking until the gate opens. + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(self.both_loaded.wait(), timeout=self.GATE_TIMEOUT) + return snapshot + + async def save_state( + self, session: AgentSession, items: list[TodoItem], *, next_id: int, source_id: str + ) -> None: + self.items = list(items) + self.next_id = next_id + + session = AgentSession(session_id="session-1") + store = _GatedSharedTodoStore() + add_tools = [] + for _ in range(2): + agent = Agent(client=chat_client_base, context_providers=[TodoProvider(store=store)]) + _, options = await agent._prepare_session_and_messages( # pyright: ignore[reportPrivateUsage] + session=session, + input_messages=[Message(role="user", contents=["Track this work"])], + ) + tools = options["tools"] + assert isinstance(tools, list) + add_tools.append(_tool_by_name(tools, "todos_add")) + + # Preparation above loads the todo list once per provider; gate only the mutations. + store.reset_gate() + + results = await asyncio.gather( + add_tools[0].invoke(arguments={"todos": [{"title": "From A"}]}), # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + add_tools[1].invoke(arguments={"todos": [{"title": "From B"}]}), # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + ) + + reported_ids = sorted(json.loads(result[0].text)[0]["id"] for result in results) + assert store.load_count == 2 + assert sorted(item.id for item in store.items) == [1, 2] + assert {item.title for item in store.items} == {"From A", "From B"} + assert reported_ids == [1, 2] + + +async def test_todo_providers_serialize_equivalent_sessions_over_one_file( + tmp_path: Path, chat_client_base: SupportsChatGetResponse +) -> None: + """Two sessions that resolve to the same todo file must not both start from an empty state.""" + + class _GatedFileTodoStore(TodoFileStore): + """Hold each load until a competing caller also finishes loading, so the loads interleave.""" + + GATE_TIMEOUT = 0.25 + + def __init__(self, base_path: Path) -> None: + super().__init__(base_path) + self.load_count = 0 + self.both_loaded = asyncio.Event() + + def reset_gate(self) -> None: + self.load_count = 0 + self.both_loaded.clear() + + async def load_state(self, session: AgentSession, *, source_id: str) -> tuple[list[TodoItem], int]: + result = await super().load_state(session, source_id=source_id) + self.load_count += 1 + if self.load_count >= 2: + self.both_loaded.set() + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(self.both_loaded.wait(), timeout=self.GATE_TIMEOUT) + return result + + store = _GatedFileTodoStore(tmp_path) + add_tools = [] + for _ in range(2): + agent = Agent(client=chat_client_base, context_providers=[TodoProvider(store=store)]) + _, options = await agent._prepare_session_and_messages( # pyright: ignore[reportPrivateUsage] + session=AgentSession(session_id="session-1"), + input_messages=[Message(role="user", contents=["Track this work"])], + ) + tools = options["tools"] + assert isinstance(tools, list) + add_tools.append(_tool_by_name(tools, "todos_add")) + + # Preparation above loads the todo list once per provider; gate only the mutations. + store.reset_gate() + + results = await asyncio.gather( + add_tools[0].invoke(arguments={"todos": [{"title": "From A"}]}), # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + add_tools[1].invoke(arguments={"todos": [{"title": "From B"}]}), # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + ) + + # Distinct session objects, one shared file: serialized mutations assign unique IDs. + reported_ids = sorted(json.loads(result[0].text)[0]["id"] for result in results) + assert reported_ids == [1, 2] + + +async def test_todo_mutation_keeps_lock_until_cancelled_write_finishes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, chat_client_base: SupportsChatGetResponse +) -> None: + """A cancelled mutation must not release the lock while its write is still in flight.""" + store = TodoFileStore(tmp_path) + provider = TodoProvider(store=store) + session = AgentSession(session_id="session-1") + agent = Agent(client=chat_client_base, context_providers=[provider]) + _, options = await agent._prepare_session_and_messages( # pyright: ignore[reportPrivateUsage] + session=session, + input_messages=[Message(role="user", contents=["Track this work"])], + ) + tools = options["tools"] + assert isinstance(tools, list) + add_todos = _tool_by_name(tools, "todos_add") + + write_started = threading.Event() + release_write = threading.Event() + real_save = TodoFileStore._save_state_sync + + def _gated_save_sync(state_path: Path, payload: str) -> None: + write_started.set() + release_write.wait(timeout=5) + real_save(state_path, payload) + + monkeypatch.setattr(TodoFileStore, "_save_state_sync", staticmethod(_gated_save_sync)) + + first = asyncio.create_task( + add_todos.invoke(arguments={"todos": [{"title": "Survives"}]}) # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + ) + await asyncio.to_thread(write_started.wait, 5) + first.cancel() + # Give the cancellation a chance to (wrongly) release the lock while the write runs. + await asyncio.sleep(0.1) + + second = asyncio.create_task( + add_todos.invoke(arguments={"todos": [{"title": "After"}]}) # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + ) + await asyncio.sleep(0.1) + assert not second.done(), "the second mutation must wait for the in-flight write to finish" + + release_write.set() + with contextlib.suppress(asyncio.CancelledError): + await first + await second + + items, _ = await store.load_state(session, source_id=provider.source_id) + assert [item.title for item in items] == ["Survives", "After"] + assert [item.id for item in items] == [1, 2] + + def test_todo_harness_graduated_classes_are_not_experimental() -> None: """Graduated todo harness types should carry no experimental metadata; TodoFileStore stays experimental.""" for graduated in (TodoStore, TodoItem, TodoInput, TodoSessionStore, TodoProvider):