diff --git a/python/packages/monty/AGENTS.md b/python/packages/monty/AGENTS.md index 8ad4c8bf8b9..9b8befa3eb1 100644 --- a/python/packages/monty/AGENTS.md +++ b/python/packages/monty/AGENTS.md @@ -36,7 +36,9 @@ from agent_framework.monty import ( - `file_mounts` — sequence of `FileMountInput` (str shorthand, `(host_path, mount_path)` tuple, or `FileMount`) - `resource_limits` — Monty 1.x `ResourceLimits` keys; legacy `max_duration_secs` - maps to `max_feed_duration_secs`, and supplying both is an error + maps to `max_feed_duration_secs`, and supplying both is an error. When neither + key is set, a 30-second `max_feed_duration_secs` applies at run time (it is not + part of the `resource_limits` property or serializable state); `None` removes it. - `include_detailed_errors` — keyword-only Boolean, default `False`. Host failures become generic `RuntimeError`s and execution errors omit raw details. Trusted opt-in preserves original diagnostics. The flag is not a code input, is independent of outer-client error/logging settings, and is copied @@ -72,9 +74,26 @@ Tool-management methods on both classes: `add_tools`, `get_tools`, - **`_execute_code_tool.py`** — `MontyExecuteCodeTool` plus tool / mount normalization, approval helpers, dynamic `description`/`instructions` builders, and the post-execution file-capture flow that surfaces files - written to `read-write` mounts as `Content.from_data` items. + written to `read-write` mounts as `Content.from_data` items. Captured files + are attributed by comparing the mount before and after a run, so calls on a + tool with a `read-write` mount take turns through a `MountGate`. The tool + hands its gate to every run-scoped copy from `create_run_tool()`, so all runs + of one provider share it. Tools and mounts are read once a call has its turn. +- **`_mount_gate.py`** — `MountGate`, a first-in-first-out turn gate that works + across event loops and threads (a lock bound to one event loop would not). A + call that a host tool starts while its own call has the turn is part of that + turn and does not wait (tracked with a context variable, so it needs no + explicit hand-off). A counter on the turn tracks its running calls, and the + gate passes the turn on only when the last one has ended, even if a host tool + left a call running in the background. A waiting call fails with + `MountGateTimeoutError` after `WAIT_TIMEOUT_SECS`, which `_run_code` turns into + an error `Content` whose message always shows. The gate does not cap how long + a call holds its turn. A copy of a tool shares the gate, and a pickled tool + gets a fresh one. - **`_monty_bridge.py`** — `InlineCodeBridge` and `generate_type_stubs`, - adapted from the reference Monty CodeAct repo. Pauses on `FunctionSnapshot` + adapted from the reference Monty CodeAct repo. Runs guest code through the + async pool (`AsyncMonty`) so it never blocks the event loop, and awaits every + `feed_start` / `resume` / `resume_auto`. Pauses on `AsyncFunctionSnapshot` to dispatch host calls, then resumes; supports direct typed tool calls (keyword arguments only; the generated stubs are keyword-only, omit parameters guest code cannot supply such as an injected context, `self`, `*args` and diff --git a/python/packages/monty/README.md b/python/packages/monty/README.md index bcf6c3fcfc3..b72bd5148a7 100644 --- a/python/packages/monty/README.md +++ b/python/packages/monty/README.md @@ -234,6 +234,28 @@ codeact = MontyCodeActProvider( `max_memory`, `gc_interval`, `max_recursion_depth`, `max_suspensions`, and `max_total_sleep_secs`. These interpreter limits do not independently bound host-side post-execution attachment capture. +- Sandbox code runs in Monty worker processes, so it does not block the event + loop, and cancelling an `execute_code` call discards its worker. When + `resource_limits` sets neither `max_feed_duration_secs` nor `max_duration_secs`, + a **30-second** `max_feed_duration_secs` applies. Set it to another value, or to + `None` (`resource_limits={"max_feed_duration_secs": None}`), to change or remove + the limit. A call that reaches the limit returns an `Execution error`; with + `include_detailed_errors=True` its details name the limit + (`TimeoutError: feed time limit exceeded`). Concurrent `execute_code` calls + overlap and each starts its own worker process, so bound concurrency in your + application when serving many callers. +- Calls that can write to the same host files take turns, so captured files stay + attributed to the right call. A tool with a `read-write` mount (including + `workspace_root`) serves one call at a time. This covers every run of one + provider and works across event loops and threads. A call waits at most 10 + minutes for its turn and then returns an error, and a slow host tool holds up + the calls behind it. A call that a host tool starts while its own call has the + turn, such as a sub-agent that uses the same provider, is part of that call + and does not wait; the outer call also returns the files that the inner call + wrote. The turn lasts until the call and every call it started have ended, + even when a host tool leaves one running in the background. Separate tools or + providers that mount the same host directory do not take turns with each + other, so give each one its own directory. ## DSL inside `execute_code` diff --git a/python/packages/monty/agent_framework_monty/_execute_code_tool.py b/python/packages/monty/agent_framework_monty/_execute_code_tool.py index 0cb4923d230..d639eb68229 100644 --- a/python/packages/monty/agent_framework_monty/_execute_code_tool.py +++ b/python/packages/monty/agent_framework_monty/_execute_code_tool.py @@ -34,6 +34,7 @@ from ._instructions import build_codeact_instructions, build_execute_code_description from ._monty_bridge import InlineCodeBridge, generate_type_stubs +from ._mount_gate import MountGate, MountGateTimeoutError from ._types import FileMount, FileMountInput logger = logging.getLogger("agent_framework") @@ -249,7 +250,12 @@ class MontyExecuteCodeTool(FunctionTool): Hyperlight's ``/output`` flow. ``resource_limits`` is forwarded to Monty's ``ResourceLimits`` to cap CPU - time, memory, output size, recursion depth, and GC frequency. + time, memory, output size, recursion depth, and GC frequency. When it sets + neither ``max_feed_duration_secs`` nor its legacy alias ``max_duration_secs``, + guest execution is limited to 30 seconds, not counting time spent waiting on + host tools. Set ``max_feed_duration_secs`` to another value, or to ``None`` to + remove the limit. ``max_duration_secs`` will be removed soon, because this + package is in beta. ``tool_description_format`` controls parameter documentation in both the description and instructions: ``"compact"`` (default) or ``"json"`` globally, @@ -258,8 +264,21 @@ class MontyExecuteCodeTool(FunctionTool): are copied and retain entries for tools registered later. All mutators (``add_tools``, ``add_file_mounts`` etc.) must be called from - the same task/thread that owns the tool. Monty itself runs on the event - loop, so no internal locking is needed. + the same task/thread that owns the tool; they are not synchronized. Guest + code runs in Monty worker processes and does not block the event loop, so + concurrent ``execute_code`` calls overlap and each starts its own worker + process. + + Calls on a tool that has a read-write mount (including ``workspace_root``) + take turns, so that written files are attributed to the right call. This + covers every run of one provider and works across event loops and threads. + A call waits at most 10 minutes for its turn and then returns an error. A call + that a host tool starts while its own call has the turn, such as a sub-agent + that uses the same provider, is part of that call and does not wait. The turn + lasts until the call and every call it started have ended. Tools and mounts + are read when a call gets its turn. Separate tools or providers that mount the + same host directory do not take turns with each other, so give each one its + own directory. Keyword Args: include_detailed_errors: Include host exception types/messages and execution error details. @@ -278,6 +297,7 @@ def __init__( tool_description_format: _ToolDescriptionFormat = "compact", include_detailed_errors: bool = False, _fides_warned_metadata: list[tuple[FunctionTool, dict[str, str]]] | None = None, + _mount_gate: MountGate | None = None, ) -> None: if not isinstance(include_detailed_errors, bool): raise TypeError("include_detailed_errors must be a bool.") @@ -300,6 +320,7 @@ def __init__( ) self._file_mounts: dict[str, FileMount] = {} self._resource_limits: dict[str, Any] | None = dict(resource_limits) if resource_limits else None + self._mount_gate: MountGate = _mount_gate or MountGate() if tools is not None: self.add_tools(tools) @@ -398,7 +419,10 @@ def workspace_root(self) -> Path | None: @property def resource_limits(self) -> dict[str, Any] | None: - """Return the configured Monty :class:`pydantic_monty.ResourceLimits`, if any.""" + """Return the configured Monty :class:`pydantic_monty.ResourceLimits`, if any. + + The default guest execution limit applied when no feed duration is configured is not included. + """ return dict(self._resource_limits) if self._resource_limits else None def build_instructions(self, *, tools_visible_to_model: bool) -> str: @@ -423,6 +447,7 @@ def create_run_tool(self) -> MontyExecuteCodeTool: resource_limits=self._resource_limits, tool_description_format=self._tool_description_format, include_detailed_errors=self._include_detailed_errors, + _mount_gate=self._mount_gate, ) def build_serializable_state(self) -> dict[str, Any]: @@ -500,6 +525,17 @@ def _effective_mounts(self) -> list[FileMount]: return mounts async def _run_code(self, *, code: str) -> list[Content]: + # Written files are attributed by comparing read-write mounts before and after the run, so calls that can + # write to the same host files take turns. + if not any(mount.mode == "read-write" for mount in self._effective_mounts()): + return await self._execute(code) + try: + async with self._mount_gate.hold(): + return await self._execute(code) + except MountGateTimeoutError as exc: + return [Content.from_error(message=str(exc))] + + async def _execute(self, code: str) -> list[Content]: include_detailed_errors = self._include_detailed_errors tools = list(self._managed_tools) mounts = self._effective_mounts() diff --git a/python/packages/monty/agent_framework_monty/_monty_bridge.py b/python/packages/monty/agent_framework_monty/_monty_bridge.py index 80139ce6562..d2293f5c9e5 100644 --- a/python/packages/monty/agent_framework_monty/_monty_bridge.py +++ b/python/packages/monty/agent_framework_monty/_monty_bridge.py @@ -17,6 +17,9 @@ MAX_PRINT_OUTPUT_CHARS = 8192 +# Guest execution budget in seconds, applied when ``resource_limits`` configures no feed duration. +DEFAULT_MAX_FEED_DURATION_SECS = 30.0 + # Prelude injected into all Monty code so `asyncio.gather` works for fan-out. _CODEACT_PRELUDE = """\ import asyncio @@ -203,8 +206,8 @@ def output(self) -> str: def load_monty() -> Any: """Import ``pydantic_monty`` lazily so unit tests can run without it. - Returns the module so callers can read ``Monty``, ``MontyComplete``, - ``FunctionSnapshot``, ``FutureSnapshot``, ``NameLookupSnapshot`` from it. + Returns the module so callers can read ``AsyncMonty``, ``MontyComplete``, + ``AsyncFunctionSnapshot``, ``AsyncFutureSnapshot``, ``AsyncNameLookupSnapshot`` from it. """ try: import pydantic_monty @@ -220,8 +223,14 @@ class InlineCodeBridge: """Execute Monty code inline (non-durable). Supports both ``await call_tool('name', ...)`` and direct ``await name(...)`` - calls. When Monty yields a :class:`FutureSnapshot`, the bridge invokes the + calls. When Monty yields an :class:`AsyncFutureSnapshot`, the bridge invokes the registered host tools and resumes execution with the results. + + Guest code runs in Monty worker processes through the asynchronous pool API, so it does + not block the event loop. Cancelling :meth:`run` discards the worker. When ``resource_limits`` + configures neither ``max_feed_duration_secs`` nor the legacy ``max_duration_secs``, + ``DEFAULT_MAX_FEED_DURATION_SECS`` bounds the guest execution time. Set + ``max_feed_duration_secs`` to ``None`` to remove that bound. """ def __init__( @@ -245,11 +254,11 @@ async def run(self, code: str) -> dict[str, Any]: raise ValueError("Code must be a non-empty string.") monty_module = load_monty() - Monty = monty_module.Monty + AsyncMonty = monty_module.AsyncMonty MontyComplete = monty_module.MontyComplete - FunctionSnapshot = monty_module.FunctionSnapshot - FutureSnapshot = monty_module.FutureSnapshot - NameLookupSnapshot = monty_module.NameLookupSnapshot + FunctionSnapshot = monty_module.AsyncFunctionSnapshot + FutureSnapshot = monty_module.AsyncFutureSnapshot + NameLookupSnapshot = monty_module.AsyncNameLookupSnapshot printer = _PrintCollector() checkout_kwargs: dict[str, Any] = { @@ -257,20 +266,21 @@ async def run(self, code: str) -> dict[str, Any]: "type_check": self.type_stubs is not None, "type_check_stubs": self.type_stubs, } - if self._resource_limits: - limits = dict(self._resource_limits) - if "max_duration_secs" in limits: - if "max_feed_duration_secs" in limits: - raise ValueError("Specify only one of max_duration_secs and max_feed_duration_secs.") - limits["max_feed_duration_secs"] = limits.pop("max_duration_secs") - checkout_kwargs["limits"] = limits + limits = dict(self._resource_limits) if self._resource_limits else {} + if "max_duration_secs" in limits: + if "max_feed_duration_secs" in limits: + raise ValueError("Specify only one of max_duration_secs and max_feed_duration_secs.") + limits["max_feed_duration_secs"] = limits.pop("max_duration_secs") + # Only an absent key gets the default; an explicit ``None`` is the caller's opt-out. + limits.setdefault("max_feed_duration_secs", DEFAULT_MAX_FEED_DURATION_SECS) + checkout_kwargs["limits"] = limits feed_kwargs: dict[str, Any] = {"print_callback": printer} if self._mounts: feed_kwargs["mount"] = list(self._mounts) - with Monty() as pool, pool.checkout(**checkout_kwargs) as session: - progress = session.feed_start(_build_code(code), **feed_kwargs) + async with AsyncMonty() as pool, pool.checkout(**checkout_kwargs) as session: + progress = await session.feed_start(_build_code(code), **feed_kwargs) while True: if isinstance(progress, MontyComplete): @@ -280,7 +290,7 @@ async def run(self, code: str) -> dict[str, Any]: "truncated": printer.truncated, } if isinstance(progress, FunctionSnapshot): - progress = self._handle_function(progress) + progress = await self._handle_function(progress) continue if isinstance(progress, FutureSnapshot): progress = await self._handle_future(progress) @@ -289,14 +299,14 @@ async def run(self, code: str) -> dict[str, Any]: raise RuntimeError(f"Name lookup not supported: {progress.variable_name!r}") raise RuntimeError(f"Unsupported Monty progress type: {type(progress).__name__}") - def _handle_function(self, snapshot: Any) -> Any: + async def _handle_function(self, snapshot: Any) -> Any: if snapshot.is_os_function: # pydantic-monty pool API: feed_start always surfaces OS calls as # snapshots. Mounts (and os=) are consulted only by resume_auto(), # not by a plain resume(...) payload. if self._mounts: - return snapshot.resume_auto() - return snapshot.resume({ + return await snapshot.resume_auto() + return await snapshot.resume({ "exc_type": "PermissionError", "message": "OS and filesystem calls are not available.", }) @@ -304,31 +314,31 @@ def _handle_function(self, snapshot: Any) -> Any: function_name = str(snapshot.function_name) if function_name in self.tool_map: - return self._schedule_direct_tool(snapshot, function_name) + return await self._schedule_direct_tool(snapshot, function_name) if function_name == "call_tool": - return self._schedule_call_tool(snapshot) + return await self._schedule_call_tool(snapshot) - return snapshot.resume({ + return await snapshot.resume({ "exc_type": "NameError", "message": f"Function {function_name!r} is not available.", }) - def _schedule_direct_tool(self, snapshot: Any, name: str) -> Any: + async def _schedule_direct_tool(self, snapshot: Any, name: str) -> Any: # The generated stubs are keyword-only, so ``ty`` rejects positional calls before any code runs. Fail # closed on anything that still reaches the host (for example when type checking is bypassed) instead of # dropping the positional arguments and running the host tool with its defaults. This is a bridge-owned # usage error, not a host failure, so it stays actionable under redaction (like ``_schedule_call_tool``). if snapshot.args: - return snapshot.resume( + return await snapshot.resume( _external_error( TypeError(f"{name} accepts keyword arguments only. Use keyword arguments for parameters."), include_detailed_errors=True, ) ) self._pending_calls[int(snapshot.call_id)] = (name, dict(snapshot.kwargs)) - return snapshot.resume({"future": ...}) + return await snapshot.resume({"future": ...}) - def _schedule_call_tool(self, snapshot: Any) -> Any: + async def _schedule_call_tool(self, snapshot: Any) -> Any: try: name, kwargs = _parse_call_tool(snapshot.args, snapshot.kwargs) if name not in self.tool_map: @@ -336,13 +346,13 @@ def _schedule_call_tool(self, snapshot: Any) -> Any: raise ValueError(f"Tool {name!r} is not registered. Available tools: {allowed}") self._pending_calls[int(snapshot.call_id)] = (name, kwargs) except Exception as exc: - return snapshot.resume(_external_error(exc, include_detailed_errors=True)) - return snapshot.resume({"future": ...}) + return await snapshot.resume(_external_error(exc, include_detailed_errors=True)) + return await snapshot.resume({"future": ...}) async def _handle_future(self, snapshot: Any) -> Any: pending_call_ids = [int(cid) for cid in snapshot.pending_call_ids] if not pending_call_ids: - return snapshot.resume({}) + return await snapshot.resume({}) entries: list[tuple[int, tuple[str, dict[str, Any]]]] = [] for cid in pending_call_ids: @@ -353,7 +363,7 @@ async def _handle_future(self, snapshot: Any) -> Any: tasks = [self._invoke_tool(cid, name, kwargs) for cid, (name, kwargs) in entries] results = await asyncio.gather(*tasks) resume_results: dict[int, Any] = dict(results) - return snapshot.resume(resume_results) + return await snapshot.resume(resume_results) async def _invoke_tool(self, cid: int, name: str, kwargs: dict[str, Any]) -> tuple[int, Any]: # Every entry in ``self.tool_map`` is produced by ``_make_tool_callback`` diff --git a/python/packages/monty/agent_framework_monty/_mount_gate.py b/python/packages/monty/agent_framework_monty/_mount_gate.py new file mode 100644 index 00000000000..abc4b57090d --- /dev/null +++ b/python/packages/monty/agent_framework_monty/_mount_gate.py @@ -0,0 +1,138 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Lets ``execute_code`` calls that can write to the same host files take turns.""" + +from __future__ import annotations + +import asyncio +import collections +import contextlib +import threading +from collections.abc import AsyncGenerator +from contextvars import ContextVar +from typing import Any + +#: How long a call waits for its turn before it fails, in seconds. +WAIT_TIMEOUT_SECS = 600.0 + + +class MountGateTimeoutError(TimeoutError): + """Raised when a call waits too long for its turn.""" + + +class _Hold: + """One turn: the call that has it, plus the calls started inside it that are still running.""" + + __slots__ = ("calls", "gate") + + def __init__(self, gate: MountGate) -> None: + self.gate = gate + self.calls = 1 + + +_HELD: ContextVar[tuple[_Hold, ...]] = ContextVar("agent_framework_monty_held_mount_gates", default=()) + + +def _grant(waiter: asyncio.Future[None]) -> None: + if not waiter.done(): + waiter.set_result(None) + + +def _expire(waiter: asyncio.Future[None], timeout: float) -> None: + if not waiter.done(): + waiter.set_exception( + MountGateTimeoutError( + f"Timed out after {timeout:g} seconds waiting for another execute_code call " + "that uses the same writable mount." + ) + ) + + +class MountGate: + """Serializes calls that can write to the same host files. + + Waiting calls are served in arrival order. The gate works across event loops and threads. A call that a host + tool makes while its own call holds the gate (for example a sub-agent that uses the same provider) is part of + that call's turn, so it does not wait. The turn lasts until the call and every call started inside it have + ended, even when a host tool leaves one running in the background. + """ + + def __init__(self) -> None: + self._state = threading.Lock() + self._held = False + self._waiters: collections.deque[tuple[asyncio.AbstractEventLoop, asyncio.Future[None]]] = collections.deque() + + def __deepcopy__(self, memo: dict[int, Any]) -> MountGate: + # A copy of a tool uses the same host files, so it takes turns with the original. + return self + + def __reduce__(self) -> tuple[type[MountGate], tuple[()]]: + # The queue holds event-loop objects that cannot be pickled, so an unpickled tool starts with a fresh gate. + return (MountGate, ()) + + @contextlib.asynccontextmanager + async def hold(self) -> AsyncGenerator[None]: + """Wait for a turn and keep it until the block and every call started inside it have ended.""" + entry = self._join_turn() + token = None + if entry is None: + await self._acquire() + entry = _Hold(self) + token = _HELD.set((*_HELD.get(), entry)) + try: + yield + finally: + if token is not None: + _HELD.reset(token) + with self._state: + entry.calls -= 1 + last = entry.calls == 0 + if last: + self._release() + + def _join_turn(self) -> _Hold | None: + # A call that a running call started belongs to that call's turn. A turn whose calls have all ended is over. + with self._state: + for entry in reversed(_HELD.get()): + if entry.gate is self and entry.calls: + entry.calls += 1 + return entry + return None + + async def _acquire(self) -> None: + loop = asyncio.get_running_loop() + with self._state: + # While the gate is free nobody is queued, because a release hands the turn directly to the next waiter. + if not self._held: + self._held = True + return + waiter: asyncio.Future[None] = loop.create_future() + entry = (loop, waiter) + self._waiters.append(entry) + timeout = WAIT_TIMEOUT_SECS + timer = loop.call_later(timeout, _expire, waiter, timeout) + try: + await waiter + except BaseException: + with self._state: + handed_over = entry not in self._waiters + if not handed_over: + self._waiters.remove(entry) + if handed_over: + # A release already gave this call the turn, so pass it on instead of losing it. + self._release() + raise + finally: + timer.cancel() + + def _release(self) -> None: + with self._state: + while self._waiters: + loop, waiter = self._waiters.popleft() + try: + loop.call_soon_threadsafe(_grant, waiter) + except RuntimeError: + # That loop is closed, so its waiter is gone. + continue + return + self._held = False diff --git a/python/packages/monty/agent_framework_monty/_provider.py b/python/packages/monty/agent_framework_monty/_provider.py index 4b4a79fb90d..30425e3203a 100644 --- a/python/packages/monty/agent_framework_monty/_provider.py +++ b/python/packages/monty/agent_framework_monty/_provider.py @@ -23,7 +23,10 @@ class MontyCodeActProvider(ContextProvider): Mirrors :class:`agent_framework_hyperlight.HyperlightCodeActProvider` for the subset of capabilities that apply to the Monty interpreter: ``tools``, ``approval_mode``, ``workspace_root``, ``file_mounts``, and - ``resource_limits`` (Monty-only). + ``resource_limits`` (Monty-only). ``resource_limits`` follows + :class:`MontyExecuteCodeTool`, including its default 30-second guest + execution limit. Calls that can write to the same host files take turns + across every run of the provider, as described for :class:`MontyExecuteCodeTool`. ``tool_description_format`` controls parameter documentation in both injected instructions and the tool description. Pass ``"compact"`` (the diff --git a/python/packages/monty/tests/monty/test_monty_codeact.py b/python/packages/monty/tests/monty/test_monty_codeact.py index 94e2a2a314f..3900297d2da 100644 --- a/python/packages/monty/tests/monty/test_monty_codeact.py +++ b/python/packages/monty/tests/monty/test_monty_codeact.py @@ -53,11 +53,11 @@ class _FakeFunctionSnapshot: is_os_function: bool = False _script: _FakeScript | None = None - def resume(self, payload: Any) -> Any: + async def resume(self, payload: Any) -> Any: assert self._script is not None, "Snapshot must be attached to a script." return self._script.advance(("function_resume", self, payload)) - def resume_auto(self) -> Any: + async def resume_auto(self) -> Any: assert self._script is not None, "Snapshot must be attached to a script." return self._script.advance(("function_resume_auto", self, None)) @@ -67,7 +67,7 @@ class _FakeFutureSnapshot: pending_call_ids: list[int] _script: _FakeScript | None = None - def resume(self, payload: Any) -> Any: + async def resume(self, payload: Any) -> Any: assert self._script is not None, "Snapshot must be attached to a script." return self._script.advance(("future_resume", self, payload)) @@ -127,8 +127,11 @@ def _get_script() -> _FakeScript: return script +_checkouts: list[_FakeSession] = [] + + class _FakeSession: - """Fake ``MontySession`` matching the pydantic-monty pool/checkout API.""" + """Fake ``AsyncMontySession`` matching the pydantic-monty async pool/checkout API.""" def __init__( self, @@ -145,13 +148,13 @@ def __init__( self.code: str | None = None self._script = _get_script() - def __enter__(self) -> _FakeSession: + async def __aenter__(self) -> _FakeSession: return self - def __exit__(self, *args: Any) -> None: + async def __aexit__(self, *args: Any) -> None: return None - def feed_start( + async def feed_start( self, code: str, *, @@ -171,16 +174,16 @@ def feed_start( class _FakeMonty: - """Fake ``Monty`` pool: ``with Monty() as pool: with pool.checkout() as session``.""" + """Fake ``AsyncMonty`` pool: ``async with AsyncMonty() as pool, pool.checkout() as session``.""" def __init__(self, **kwargs: Any) -> None: self.pool_kwargs = kwargs self.last_session: _FakeSession | None = None - def __enter__(self) -> _FakeMonty: + async def __aenter__(self) -> _FakeMonty: return self - def __exit__(self, *args: Any) -> None: + async def __aexit__(self, *args: Any) -> None: return None def checkout( @@ -198,6 +201,7 @@ def checkout( type_check_stubs=type_check_stubs, limits=limits, ) + _checkouts.append(self.last_session) return self.last_session @@ -205,16 +209,18 @@ def checkout( def fake_monty_module(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: """Install a fake ``pydantic_monty`` module for the duration of each test.""" fake = types.ModuleType("pydantic_monty") - fake.Monty = _FakeMonty # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + fake.AsyncMonty = _FakeMonty # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] fake.MontyComplete = _FakeMontyComplete # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] - fake.FunctionSnapshot = _FakeFunctionSnapshot # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] - fake.FutureSnapshot = _FakeFutureSnapshot # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] - fake.NameLookupSnapshot = _FakeNameLookupSnapshot # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + fake.AsyncFunctionSnapshot = _FakeFunctionSnapshot # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + fake.AsyncFutureSnapshot = _FakeFutureSnapshot # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + fake.AsyncNameLookupSnapshot = _FakeNameLookupSnapshot # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] monkeypatch.setitem(sys.modules, "pydantic_monty", fake) _current_script[0] = None + _checkouts.clear() yield _current_script[0] = None + _checkouts.clear() # --------------------------------------------------------------------------- @@ -743,6 +749,33 @@ def test_resource_limits_round_trip() -> None: assert state["resource_limits"] == {"max_duration_secs": 5.0} +@pytest.mark.parametrize( + ("resource_limits", "expected_limits"), + [ + (None, {"max_feed_duration_secs": 30.0}), + ({}, {"max_feed_duration_secs": 30.0}), + ({"max_memory": 1024}, {"max_memory": 1024, "max_feed_duration_secs": 30.0}), + ({"max_turn_duration_secs": 2.0}, {"max_turn_duration_secs": 2.0, "max_feed_duration_secs": 30.0}), + ({"max_feed_duration_secs": 5.0}, {"max_feed_duration_secs": 5.0}), + ({"max_duration_secs": 5.0}, {"max_feed_duration_secs": 5.0}), + ({"max_feed_duration_secs": None}, {"max_feed_duration_secs": None}), + ({"max_duration_secs": None}, {"max_feed_duration_secs": None}), + ], +) +async def test_guest_execution_limit_defaults_unless_a_feed_duration_is_configured( + resource_limits: dict[str, Any] | None, expected_limits: dict[str, Any] +) -> None: + _set_script(_FakeMontyComplete(output=None)) + monty_tool = MontyExecuteCodeTool(resource_limits=resource_limits) + + await monty_tool._run_code(code="None") + + assert _checkouts[-1].limits == expected_limits + # The default applies at run time only; configuration and serialized state keep what the caller supplied. + assert monty_tool.resource_limits == (resource_limits or None) + assert monty_tool.build_serializable_state()["resource_limits"] == (resource_limits or None) + + def test_build_instructions_includes_registered_tools() -> None: monty_tool = MontyExecuteCodeTool(tools=[add_tool]) instructions = monty_tool.build_instructions(tools_visible_to_model=False) diff --git a/python/packages/monty/tests/monty/test_monty_runtime.py b/python/packages/monty/tests/monty/test_monty_runtime.py index a166a2a7ec9..6171c14ef89 100644 --- a/python/packages/monty/tests/monty/test_monty_runtime.py +++ b/python/packages/monty/tests/monty/test_monty_runtime.py @@ -4,16 +4,28 @@ from __future__ import annotations +import asyncio +import contextlib +from pathlib import Path from typing import Any import pytest -from agent_framework import FunctionTool -from pydantic_monty import MontyRuntimeError +from agent_framework import Content, FunctionTool +from pydantic_monty import MontyRuntimeError, MountDir from agent_framework_monty import MontyExecuteCodeTool +from agent_framework_monty import _monty_bridge as bridge_module +from agent_framework_monty import _mount_gate as gate_module from agent_framework_monty._execute_code_tool import _make_tool_callback from agent_framework_monty._monty_bridge import InlineCodeBridge +_RUNAWAY_GUEST = "while True:\n pass" +_READ_MOUNT_FILE = "from pathlib import Path\nPath('/input/value.txt').read_text()\n" +_READ_MOUNT_FILE_IGNORING_ERRORS = ( + "from pathlib import Path\ntry:\n Path('/input/value.txt').read_text()\nexcept Exception:\n pass\n" +) +_WRITE_FILE_THEN_PAUSE = "from pathlib import Path\nPath('/input/{name}.txt').write_text('{name}')\nawait pause()\n" + @pytest.mark.parametrize( "resource_limits", @@ -64,6 +76,216 @@ async def test_legacy_and_feed_duration_limits_are_rejected_as_ambiguous(legacy_ await bridge.run("print('must not run')") +async def test_default_feed_duration_limit_stops_runaway_guest_code(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(bridge_module, "DEFAULT_MAX_FEED_DURATION_SECS", 0.3) + + with pytest.raises(MontyRuntimeError, match="feed time limit exceeded"): + await InlineCodeBridge({}).run(_RUNAWAY_GUEST) + + +@pytest.mark.parametrize( + ("code", "use_mount"), + [ + (_RUNAWAY_GUEST, False), + (f"await record()\n{_RUNAWAY_GUEST}", False), + (f"{_READ_MOUNT_FILE_IGNORING_ERRORS}{_RUNAWAY_GUEST}", False), + (f"{_READ_MOUNT_FILE}{_RUNAWAY_GUEST}", True), + ], + ids=["feed", "resume_after_host_tool", "resume_after_os_error", "resume_after_mount_read"], +) +async def test_runaway_guest_code_does_not_block_the_event_loop(code: str, use_mount: bool, tmp_path: Path) -> None: + (tmp_path / "value.txt").write_text("value", encoding="utf-8") + mounts = [MountDir(host_path=str(tmp_path), virtual_path="/input", mode="read-only")] if use_mount else None + loop = asyncio.get_running_loop() + timestamps = [loop.time()] + + async def record() -> None: + # Measure only the time after the guest resumes from this host call. + timestamps[:] = [loop.time()] + + async def heartbeat() -> None: + while True: + await asyncio.sleep(0.01) + timestamps.append(loop.time()) + + bridge = InlineCodeBridge({"record": record}, mounts=mounts, resource_limits={"max_feed_duration_secs": 1.0}) + heartbeat_task = asyncio.create_task(heartbeat()) + try: + with pytest.raises(MontyRuntimeError, match="feed time limit exceeded"): + await bridge.run(code) + finally: + heartbeat_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await heartbeat_task + timestamps.append(loop.time()) + + # A blocked loop leaves a gap about as long as the 1 s guest budget. + assert max(later - earlier for earlier, later in zip(timestamps, timestamps[1:])) < 0.5 + + +async def test_timeout_around_running_guest_code_returns_promptly_and_later_runs_recover() -> None: + guest_started = asyncio.Event() + + async def record() -> None: + guest_started.set() + + bridge = InlineCodeBridge({"record": record}, resource_limits={"max_feed_duration_secs": 2.0}) + run = asyncio.create_task(bridge.run(f"await record()\n{_RUNAWAY_GUEST}")) + await asyncio.wait_for(guest_started.wait(), timeout=30) + started = asyncio.get_running_loop().time() + + with pytest.raises(TimeoutError): + await asyncio.wait_for(run, timeout=0.3) + + assert run.cancelled() + assert asyncio.get_running_loop().time() - started < 1.5 + assert (await InlineCodeBridge({}).run("1 + 1"))["output"] == 2 + + +async def test_concurrent_runs_sharing_a_writable_mount_capture_only_their_own_files(tmp_path: Path) -> None: + async def pause() -> None: + await asyncio.sleep(0.05) + + execute_code = MontyExecuteCodeTool(tools=[FunctionTool(name="pause", func=pause)], workspace_root=tmp_path) + + first, second = await asyncio.gather( + execute_code._run_code(code=_WRITE_FILE_THEN_PAUSE.format(name="first")), + execute_code._run_code(code=_WRITE_FILE_THEN_PAUSE.format(name="second")), + ) + + assert [(c.additional_properties or {}).get("path") for c in first if c.type == "data"] == ["/input/first.txt"] + assert [(c.additional_properties or {}).get("path") for c in second if c.type == "data"] == ["/input/second.txt"] + + +async def test_concurrent_runs_without_a_writable_mount_overlap() -> None: + arrivals = 0 + both_arrived = asyncio.Event() + + async def rendezvous() -> None: + nonlocal arrivals + arrivals += 1 + if arrivals == 2: + both_arrived.set() + # Serialized runs would never meet here, so this wait would time out. + await asyncio.wait_for(both_arrived.wait(), timeout=10) + + execute_code = MontyExecuteCodeTool(tools=[FunctionTool(name="rendezvous", func=rendezvous)]) + + results = await asyncio.gather( + execute_code._run_code(code="await rendezvous()"), + execute_code._run_code(code="await rendezvous()"), + ) + + assert not any(content.type == "error" for contents in results for content in contents) + + +async def test_runs_of_one_provider_take_turns_on_a_writable_mount(tmp_path: Path) -> None: + first_in_host_tool = asyncio.Event() + release_first = asyncio.Event() + second_reached_host_tool = asyncio.Event() + + async def first_wait() -> None: + first_in_host_tool.set() + await release_first.wait() + + async def second_wait() -> None: + second_reached_host_tool.set() + + # A provider gives every agent run its own run-scoped tool. + execute_code = MontyExecuteCodeTool(tools=[first_wait, second_wait], workspace_root=tmp_path) + first_run = execute_code.create_run_tool() + second_run = execute_code.create_run_tool() + write_first = "from pathlib import Path\nPath('/input/first.txt').write_text('first')\nawait first_wait()\n" + write_second = "from pathlib import Path\nPath('/input/second.txt').write_text('second')\nawait second_wait()\n" + + first = asyncio.create_task(first_run._run_code(code=write_first)) + async with asyncio.timeout(30): + await first_in_host_tool.wait() + second = asyncio.create_task(second_run._run_code(code=write_second)) + # A run that did not wait for its turn would have reached its host tool by now. + await asyncio.sleep(0.5) + assert not second_reached_host_tool.is_set() + + release_first.set() + async with asyncio.timeout(30): + first_contents, second_contents = await asyncio.gather(first, second) + + assert [(c.additional_properties or {}).get("path") for c in first_contents if c.type == "data"] == [ + "/input/first.txt" + ] + assert [(c.additional_properties or {}).get("path") for c in second_contents if c.type == "data"] == [ + "/input/second.txt" + ] + + +async def test_host_tool_can_start_another_run_of_the_same_provider( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # A failure here would make the nested run wait for its caller, so keep it short. + monkeypatch.setattr(gate_module, "WAIT_TIMEOUT_SECS", 5.0) + execute_code = MontyExecuteCodeTool(workspace_root=tmp_path) + outer_run = execute_code.create_run_tool() + inner_run = execute_code.create_run_tool() + + async def nested() -> str: + contents = await inner_run._run_code(code="1 + 1") + return contents[0].text or str(contents[0]) + + outer_run.add_tools(FunctionTool(name="nested", func=nested)) + + async with asyncio.timeout(30): + contents = await outer_run._run_code(code="print(await nested())") + + assert [content.type for content in contents] == ["text"] + assert (contents[0].text or "").strip() == "2" + + +async def test_call_that_waits_too_long_for_its_turn_returns_an_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(gate_module, "WAIT_TIMEOUT_SECS", 0.3) + in_host_tool = asyncio.Event() + release = asyncio.Event() + + async def block() -> None: + in_host_tool.set() + await release.wait() + + execute_code = MontyExecuteCodeTool(tools=[block], workspace_root=tmp_path) + holder = asyncio.create_task(execute_code._run_code(code="await block()")) + async with asyncio.timeout(30): + await in_host_tool.wait() + waiting = await execute_code._run_code(code="1 + 1") + + assert [content.type for content in waiting] == ["error"] + assert "waiting for another execute_code call" in str(waiting[0]) + + release.set() + async with asyncio.timeout(30): + assert not any(content.type == "error" for content in await holder) + assert not any(content.type == "error" for content in await execute_code._run_code(code="1 + 1")) + + +async def test_tool_can_be_used_from_several_event_loops(tmp_path: Path) -> None: + async def pause() -> None: + await asyncio.sleep(0.05) + + execute_code = MontyExecuteCodeTool(tools=[FunctionTool(name="pause", func=pause)], workspace_root=tmp_path) + + async def two_overlapping_calls() -> tuple[list[Content], list[Content]]: + return await asyncio.gather( + execute_code._run_code(code="await pause()"), + execute_code._run_code(code="await pause()"), + ) + + def run_in_new_event_loop() -> tuple[list[Content], list[Content]]: + return asyncio.run(two_overlapping_calls()) + + results = [*await two_overlapping_calls(), *await asyncio.to_thread(run_in_new_event_loop)] + + assert not any(content.type == "error" for contents in results for content in contents) + + @pytest.mark.parametrize("recording_host_tool", ["positional_or_keyword", "keyword_only"], indirect=True) async def test_positional_tool_call_is_rejected_before_any_host_tool_runs( recording_host_tool: tuple[FunctionTool, list[str | None]], diff --git a/python/packages/monty/tests/monty/test_mount_gate.py b/python/packages/monty/tests/monty/test_mount_gate.py new file mode 100644 index 00000000000..e0ec968f0c8 --- /dev/null +++ b/python/packages/monty/tests/monty/test_mount_gate.py @@ -0,0 +1,324 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Behavior of the gate that makes calls to the same writable mount take turns.""" + +from __future__ import annotations + +import asyncio +import copy +import pickle +import threading + +import pytest + +from agent_framework_monty import _mount_gate as gate_module +from agent_framework_monty._mount_gate import MountGate, MountGateTimeoutError + + +async def test_calls_take_turns_in_arrival_order() -> None: + gate = MountGate() + order: list[str] = [] + active = 0 + peak = 0 + + async def call(name: str) -> None: + nonlocal active, peak + async with gate.hold(): + active += 1 + peak = max(peak, active) + order.append(name) + await asyncio.sleep(0.01) + active -= 1 + + tasks: list[asyncio.Task[None]] = [] + for name in ("first", "second", "third"): + tasks.append(asyncio.create_task(call(name))) + # Let each call queue before the next one starts. + await asyncio.sleep(0) + async with asyncio.timeout(10): + await asyncio.gather(*tasks) + + assert order == ["first", "second", "third"] + assert peak == 1 + + +async def test_calls_take_turns_across_event_loops_and_threads(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(gate_module, "WAIT_TIMEOUT_SECS", 30.0) + gate = MountGate() + release_first = asyncio.Event() + other_loop: list[asyncio.AbstractEventLoop] = [] + other_loop_started = threading.Event() + state_lock = threading.Lock() + entered: list[str] = [] + active = 0 + peak = 0 + + def enter(name: str) -> None: + nonlocal active, peak + with state_lock: + active += 1 + peak = max(peak, active) + entered.append(name) + + def leave() -> None: + nonlocal active + with state_lock: + active -= 1 + + async def first_call() -> None: + async with gate.hold(): + enter("first") + await release_first.wait() + leave() + + async def call_from_another_loop() -> None: + other_loop.append(asyncio.get_running_loop()) + other_loop_started.set() + async with gate.hold(): + enter("another loop") + await asyncio.sleep(0.01) + leave() + + async def later_call_on_this_loop() -> None: + async with gate.hold(): + enter("later") + leave() + + async with asyncio.timeout(60): + first = asyncio.create_task(first_call()) + await asyncio.sleep(0) + other = asyncio.create_task(asyncio.to_thread(asyncio.run, call_from_another_loop())) + await asyncio.to_thread(other_loop_started.wait, 30) + # The other loop finishes the step in which its call queued before it runs this coroutine. + await asyncio.wrap_future(asyncio.run_coroutine_threadsafe(asyncio.sleep(0), other_loop[0])) + later = asyncio.create_task(later_call_on_this_loop()) + await asyncio.sleep(0) + + release_first.set() + await asyncio.gather(first, other, later) + + assert entered == ["first", "another loop", "later"] + assert peak == 1 + + +async def test_call_fails_when_its_turn_does_not_come_in_time(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(gate_module, "WAIT_TIMEOUT_SECS", 0.2) + gate = MountGate() + release = asyncio.Event() + + async def holder() -> None: + async with gate.hold(): + await release.wait() + + holding = asyncio.create_task(holder()) + await asyncio.sleep(0) + + async with asyncio.timeout(10): + with pytest.raises(MountGateTimeoutError, match="Timed out after 0.2 seconds"): + async with gate.hold(): + pytest.fail("The turn must not be granted while another call holds it.") + release.set() + await holding + # The failed call left the queue, and the gate is free again. + async with gate.hold(): + pass + + assert not gate._waiters + + +async def test_cancelled_waiter_leaves_the_queue_without_losing_the_turn() -> None: + gate = MountGate() + release = asyncio.Event() + ran: list[str] = [] + + async def holder() -> None: + async with gate.hold(): + await release.wait() + + async def waiter(name: str) -> None: + async with gate.hold(): + ran.append(name) + + holding = asyncio.create_task(holder()) + await asyncio.sleep(0) + cancelled = asyncio.create_task(waiter("cancelled")) + survivor = asyncio.create_task(waiter("survivor")) + await asyncio.sleep(0) + cancelled.cancel() + release.set() + + async with asyncio.timeout(10): + await asyncio.gather(holding, survivor) + with pytest.raises(asyncio.CancelledError): + await cancelled + + assert ran == ["survivor"] + + +async def test_waiter_cancelled_after_the_turn_was_handed_to_it_passes_the_turn_on() -> None: + gate = MountGate() + await gate._acquire() + ran: list[str] = [] + + async def waiter(name: str) -> None: + async with gate.hold(): + ran.append(name) + + cancelled = asyncio.create_task(waiter("cancelled")) + survivor = asyncio.create_task(waiter("survivor")) + await asyncio.sleep(0) + # The release hands the turn to the first waiter, whose task has been cancelled but has not run yet. + cancelled.cancel() + gate._release() + + async with asyncio.timeout(10): + await survivor + with pytest.raises(asyncio.CancelledError): + await cancelled + + assert ran == ["survivor"] + + +async def test_release_skips_a_waiter_whose_event_loop_is_closed() -> None: + gate = MountGate() + await gate._acquire() + closed_loop = asyncio.new_event_loop() + abandoned = closed_loop.create_future() + closed_loop.close() + gate._waiters.append((closed_loop, abandoned)) + ran: list[str] = [] + + async def waiter() -> None: + async with gate.hold(): + ran.append("live") + + live = asyncio.create_task(waiter()) + await asyncio.sleep(0) + gate._release() + + async with asyncio.timeout(10): + await live + + assert ran == ["live"] + + +async def test_call_made_while_holding_the_gate_does_not_wait_for_it(monkeypatch: pytest.MonkeyPatch) -> None: + # A failure here would make the nested calls wait, so keep it short. + monkeypatch.setattr(gate_module, "WAIT_TIMEOUT_SECS", 5.0) + gate = MountGate() + + async def nested_call() -> None: + async with gate.hold(): + pass + + async with asyncio.timeout(30): + async with gate.hold(): + async with gate.hold(): + pass + await asyncio.create_task(nested_call()) + await asyncio.gather(nested_call(), nested_call()) + await asyncio.to_thread(asyncio.run, nested_call()) + + +async def test_turn_lasts_until_a_call_started_inside_it_has_ended() -> None: + gate = MountGate() + background_holds = asyncio.Event() + finish_background = asyncio.Event() + entered: list[str] = [] + + async def background_call() -> None: + async with gate.hold(): + background_holds.set() + await finish_background.wait() + + async def later_call() -> None: + async with gate.hold(): + entered.append("later") + + async with asyncio.timeout(10): + async with gate.hold(): + background = asyncio.create_task(background_call()) + await background_holds.wait() + later = asyncio.create_task(later_call()) + # The call that had the turn has ended, but the call it started has not. A call that wrongly got the turn + # would have entered by now. + await asyncio.sleep(0.05) + assert entered == [] + + finish_background.set() + async with asyncio.timeout(10): + await asyncio.gather(background, later) + + assert entered == ["later"] + + +async def test_call_started_by_a_running_call_after_the_turn_holder_ended_does_not_wait( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A failure here would make the nested call wait, so keep it short. + monkeypatch.setattr(gate_module, "WAIT_TIMEOUT_SECS", 5.0) + gate = MountGate() + background_holds = asyncio.Event() + holder_ended = asyncio.Event() + + async def background_call() -> None: + async with gate.hold(): + background_holds.set() + await holder_ended.wait() + async with gate.hold(): + pass + + async with asyncio.timeout(30): + async with gate.hold(): + background = asyncio.create_task(background_call()) + await background_holds.wait() + holder_ended.set() + await background + + +async def test_call_that_outlives_the_hold_that_started_it_waits_for_its_turn() -> None: + gate = MountGate() + proceed = asyncio.Event() + release_other = asyncio.Event() + entered: list[str] = [] + + async def late_call() -> None: + await proceed.wait() + async with gate.hold(): + entered.append("late") + + async def other_call() -> None: + async with gate.hold(): + entered.append("other") + await release_other.wait() + + async with gate.hold(): + # The task inherits the hold that is active here, which ends before the task asks for its turn. + late = asyncio.create_task(late_call()) + other = asyncio.create_task(other_call()) + await asyncio.sleep(0) + proceed.set() + # A call that wrongly skipped the wait would have entered by now. + await asyncio.sleep(0.05) + assert entered == ["other"] + + release_other.set() + async with asyncio.timeout(10): + await asyncio.gather(late, other) + + assert entered == ["other", "late"] + + +async def test_copy_shares_the_gate_and_pickle_creates_a_fresh_one() -> None: + gate = MountGate() + + async with gate.hold(): + assert copy.deepcopy(gate) is gate + # The pickle carries no state, so it also works while the gate is held. + restored = pickle.loads(pickle.dumps(gate)) + + assert isinstance(restored, MountGate) + assert restored is not gate + async with asyncio.timeout(10): + async with restored.hold(): + pass