diff --git a/python/packages/monty/AGENTS.md b/python/packages/monty/AGENTS.md index a14a77411c8..8ad4c8bf8b9 100644 --- a/python/packages/monty/AGENTS.md +++ b/python/packages/monty/AGENTS.md @@ -75,8 +75,11 @@ Tool-management methods on both classes: `add_tools`, `get_tools`, written to `read-write` mounts as `Content.from_data` items. - **`_monty_bridge.py`** — `InlineCodeBridge` and `generate_type_stubs`, adapted from the reference Monty CodeAct repo. Pauses on `FunctionSnapshot` - to dispatch host calls, then resumes; supports direct typed tool calls, - the `call_tool` fallback, `asyncio.gather` fan-out, and forwards + to dispatch host calls, then resumes; supports direct typed tool calls + (keyword arguments only; the generated stubs are keyword-only, omit parameters + guest code cannot supply such as an injected context, `self`, `*args` and + `**kwargs`, and the bridge rejects positional arguments), the `call_tool` + fallback, `asyncio.gather` fan-out, and forwards ``limits`` / type-check options via `pool.checkout(...)`, and ``mount`` / ``print_callback`` via `session.feed_start(...)`. - **`_instructions.py`** — dynamic instruction / tool-description builders diff --git a/python/packages/monty/README.md b/python/packages/monty/README.md index 483ca63278d..bcf6c3fcfc3 100644 --- a/python/packages/monty/README.md +++ b/python/packages/monty/README.md @@ -242,7 +242,7 @@ Available primitives: | Primitive | Behavior | |-----------|----------| -| `await tool_name(**kwargs)` | Direct typed call to a registered host tool. Argument types are checked before execution. | +| `await tool_name(**kwargs)` | Direct typed call to a registered host tool. Keyword arguments only; positional arguments are rejected. Argument types are checked before execution. | | `await call_tool("name", **kwargs)` | Generic fallback that dispatches by tool name. Not type-checked. | | `asyncio.gather(...)` | Fans out concurrent tool calls. | | `print(...)` | Captured and surfaced as text in the tool result. | @@ -274,6 +274,6 @@ These functions execute on the host rather than inside the Monty interpreter. `workspace_root` / `file_mounts` to grant scoped filesystem access. - Code is type-checked against tool signatures via [ty](https://docs.astral.sh/ty/) before execution, so wrong argument types - surface as a clear error before any host tool runs. + or positional-argument usage surface as a clear error before any host tool runs. - The beta package is part of `agent-framework[all]` and is reachable through the lazy-loading namespace `agent_framework.monty`. diff --git a/python/packages/monty/agent_framework_monty/_execute_code_tool.py b/python/packages/monty/agent_framework_monty/_execute_code_tool.py index fc2cbb6aa56..0cb4923d230 100644 --- a/python/packages/monty/agent_framework_monty/_execute_code_tool.py +++ b/python/packages/monty/agent_framework_monty/_execute_code_tool.py @@ -506,7 +506,8 @@ async def _run_code(self, *, code: str) -> list[Content]: tool_map = self._build_tool_map(tools) stub_map = self._build_type_stub_map(tools) - type_stubs = generate_type_stubs(stub_map) if stub_map else None + accepted_parameters = {tool_obj.name: tool_obj.parameters().get("properties", {}) for tool_obj in tools} + type_stubs = generate_type_stubs(stub_map, accepted_parameters=accepted_parameters) if stub_map else None # Snapshot mtimes of host files in read-write mounts so we can later # identify which files the sandbox actually touched. diff --git a/python/packages/monty/agent_framework_monty/_monty_bridge.py b/python/packages/monty/agent_framework_monty/_monty_bridge.py index ab387129177..80139ce6562 100644 --- a/python/packages/monty/agent_framework_monty/_monty_bridge.py +++ b/python/packages/monty/agent_framework_monty/_monty_bridge.py @@ -12,7 +12,7 @@ import keyword import types import typing -from collections.abc import Callable, Sequence +from collections.abc import Callable, Collection, Mapping, Sequence from typing import Annotated, Any, cast, get_type_hints MAX_PRINT_OUTPUT_CHARS = 8192 @@ -95,13 +95,27 @@ def _python_type_repr(annotation: Any) -> str: return str(annotation) -def generate_type_stubs(tool_callables: dict[str, Callable[..., Any]]) -> str: +def generate_type_stubs( + tool_callables: dict[str, Callable[..., Any]], + *, + accepted_parameters: Mapping[str, Collection[str]] | None = None, +) -> str: """Generate Python type stub declarations for tools + DSL primitives. Stubs are fed to Monty's ``type_check_stubs`` so ``ty`` can validate the LLM-generated code against the actual tool signatures before any host call runs. + Host tools receive keyword arguments only, so every tool parameter is + declared keyword-only regardless of how the host function declares it. + ``ty`` then rejects positional calls before any code runs instead of the + bridge silently dropping the positional arguments. ``*args`` and + ``**kwargs`` are never declared because guest code cannot supply them. + + ``accepted_parameters`` maps a tool name to the parameter names guest code + may pass (the tool's schema properties). Other parameters of the callable, + such as an injected context parameter or ``self``, are left out of the stub. + Tools whose ``name`` is not a valid Python identifier are skipped because their name cannot be safely splatted into stub source. The model can still reach them via the ``call_tool("weird name", ...)`` fallback at runtime, @@ -132,8 +146,13 @@ def generate_type_stubs(tool_callables: dict[str, Callable[..., Any]]) -> str: lines.append("") continue + accepted = accepted_parameters.get(name) if accepted_parameters is not None else None params: list[str] = [] for param_name, param in sig.parameters.items(): + if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): + continue + if accepted is not None and param_name not in accepted: + continue annotation = hints.get(param_name, inspect.Parameter.empty) type_str = _python_type_repr(annotation) if param.default is not inspect.Parameter.empty: @@ -143,7 +162,7 @@ def generate_type_stubs(tool_callables: dict[str, Callable[..., Any]]) -> str: return_annotation = hints.get("return", inspect.Parameter.empty) return_str = _python_type_repr(return_annotation) - param_str = ", ".join(params) + param_str = ", ".join(["*", *params] if params else params) lines.append(f"async def {name}({param_str}) -> {return_str}:") lines.append(" raise NotImplementedError()") lines.append("") @@ -295,10 +314,17 @@ def _handle_function(self, snapshot: Any) -> Any: }) def _schedule_direct_tool(self, snapshot: Any, name: str) -> Any: - # Positional args are rejected up-front by ``ty`` because the generated - # stubs declare every parameter as keyword-typed. Anything that slips - # through (e.g. tools with no signature inspection) is forwarded to the - # host tool as-is via kwargs only. + # The generated stubs are keyword-only, so ``ty`` rejects positional calls before any code runs. Fail + # closed on anything that still reaches the host (for example when type checking is bypassed) instead of + # dropping the positional arguments and running the host tool with its defaults. This is a bridge-owned + # usage error, not a host failure, so it stays actionable under redaction (like ``_schedule_call_tool``). + if snapshot.args: + return snapshot.resume( + _external_error( + TypeError(f"{name} accepts keyword arguments only. Use keyword arguments for parameters."), + include_detailed_errors=True, + ) + ) self._pending_calls[int(snapshot.call_id)] = (name, dict(snapshot.kwargs)) return snapshot.resume({"future": ...}) diff --git a/python/packages/monty/tests/monty/conftest.py b/python/packages/monty/tests/monty/conftest.py index d94bdb75583..79a06cd8850 100644 --- a/python/packages/monty/tests/monty/conftest.py +++ b/python/packages/monty/tests/monty/conftest.py @@ -3,9 +3,11 @@ from __future__ import annotations import asyncio +from collections.abc import Callable +from typing import Any import pytest -from agent_framework import FunctionTool +from agent_framework import FunctionInvocationContext, FunctionTool, tool @pytest.fixture @@ -35,3 +37,61 @@ async def cancel() -> str: raise asyncio.CancelledError return FunctionTool(name="cancelled_host_tool", func=cancel) + + +@pytest.fixture +def recording_host_tool(request: pytest.FixtureRequest) -> tuple[FunctionTool, list[str | None]]: + """Return a host tool and the labels it received; ``request.param`` selects the signature style.""" + calls: list[str | None] = [] + + def positional_or_keyword(label: str | None = None) -> str: + calls.append(label) + return "recorded" + + def keyword_only(*, label: str | None = None) -> str: + calls.append(label) + return "recorded" + + host_tool = FunctionTool( + name="record", + description="Record the label received by the host.", + func=keyword_only if request.param == "keyword_only" else positional_or_keyword, + ) + return host_tool, calls + + +@pytest.fixture +def unsupplied_parameter_host_tool(request: pytest.FixtureRequest) -> tuple[FunctionTool, list[str]]: + """Return a ``lookup`` host tool and the queries it received. + + ``request.param`` selects a parameter that guest code cannot supply: an injected context, ``**kwargs``, + ``*args`` or ``self``. ``FunctionTool`` leaves each of them out of the parameters it accepts. + """ + calls: list[str] = [] + + def with_context(query: str, ctx: FunctionInvocationContext) -> str: + calls.append(f"{query}:{type(ctx).__name__}") + return "recorded" + + def with_var_keyword(query: str, **extra: Any) -> str: + calls.append(query) + return "recorded" + + def with_var_positional(query: str, *extra: str) -> str: + calls.append(query) + return "recorded" + + class Service: + @tool(name="lookup") + def lookup(self, query: str) -> str: + calls.append(query) + return "recorded" + + if request.param == "method": + return Service().lookup, calls + functions: dict[str, Callable[..., Any]] = { + "context": with_context, + "var_keyword": with_var_keyword, + "var_positional": with_var_positional, + } + return FunctionTool(name="lookup", description="Record the query.", func=functions[request.param]), calls diff --git a/python/packages/monty/tests/monty/test_monty_codeact.py b/python/packages/monty/tests/monty/test_monty_codeact.py index 5a889ea6b79..94e2a2a314f 100644 --- a/python/packages/monty/tests/monty/test_monty_codeact.py +++ b/python/packages/monty/tests/monty/test_monty_codeact.py @@ -25,7 +25,7 @@ from unittest.mock import MagicMock import pytest -from agent_framework import Content, FunctionTool, Message, tool +from agent_framework import Content, FunctionInvocationContext, FunctionTool, Message, tool from agent_framework._sessions import SessionContext from agent_framework.exceptions import ToolException @@ -864,6 +864,31 @@ async def test_run_code_direct_typed_call_invokes_registered_tool() -> None: assert final_resume[1] == {"return_value": 5} +async def test_run_code_direct_typed_call_with_positional_arguments_is_rejected_without_invoking_host() -> None: + calls: list[str | None] = [] + + @tool + def record(label: str | None = None) -> str: + """Record the label received by the host.""" + calls.append(label) + return "recorded" + + func_snapshot = _FakeFunctionSnapshot(function_name="record", call_id=5, args=("positional",)) + script = _set_script(func_snapshot, _FakeMontyComplete(output=None)) + + monty_tool = MontyExecuteCodeTool(tools=[record]) + await monty_tool._run_code(code='await record("positional")') + + payloads = [payload for _, _, payload in script.resume_log] + assert payloads == [ + { + "exc_type": "TypeError", + "message": "record accepts keyword arguments only. Use keyword arguments for parameters.", + } + ] + assert calls == [] + + async def test_run_code_call_tool_fallback_invokes_registered_tool() -> None: func_snapshot = _FakeFunctionSnapshot( function_name="call_tool", @@ -1721,10 +1746,50 @@ def custom(x: int, y: str = "z") -> bool: stubs = bridge_module.generate_type_stubs({"custom": custom}) assert "async def call_tool(name: str, **kwargs: Any) -> Any:" in stubs - assert "async def custom(x: int, y: str = ...) -> bool:" in stubs + assert "async def custom(*, x: int, y: str = ...) -> bool:" in stubs assert "when_any" not in stubs +def test_generate_type_stubs_declares_every_parameter_keyword_only() -> None: + def positional_or_keyword(tenant_id: str | None = None, limit: int = 1000) -> int: + return limit + + def keyword_only(*, tenant_id: str | None = None) -> int: + return 0 + + def no_parameters() -> int: + return 0 + + stubs = bridge_module.generate_type_stubs({ + "positional_or_keyword": positional_or_keyword, + "keyword_only": keyword_only, + "no_parameters": no_parameters, + }) + + assert "async def positional_or_keyword(*, tenant_id: str | None = ..., limit: int = ...) -> int:" in stubs + assert "async def keyword_only(*, tenant_id: str | None = ...) -> int:" in stubs + # A tool without parameters keeps an empty parameter list. + assert "async def no_parameters() -> int:" in stubs + + +def test_generate_type_stubs_omits_parameters_guest_code_cannot_supply() -> None: + def wrapper(query: str, *args: int, **kwargs: Any) -> str: + return query + + def with_context(query: str, ctx: FunctionInvocationContext) -> str: + return query + + stubs = bridge_module.generate_type_stubs( + {"wrapper": wrapper, "with_context": with_context}, + accepted_parameters={"with_context": ["query"]}, + ) + + # ``*args`` and ``**kwargs`` are never declared, with or without an accepted-parameter list. + assert "async def wrapper(*, query: str) -> str:" in stubs + # Parameters outside the accepted list, such as an injected context, are left out. + assert "async def with_context(*, query: str) -> str:" in stubs + + def test_generate_type_stubs_preserves_none_and_optional() -> None: def nullable_return(x: int) -> None: @@ -1746,11 +1811,11 @@ def union_param(x: int | str | None) -> str: # noqa: UP007 - intentional }) # ``None`` return must round-trip as None, not Any. - assert "async def nullable_return(x: int) -> None:" in stubs + assert "async def nullable_return(*, x: int) -> None:" in stubs # ``Optional[X]`` is ``Union[X, None]`` at runtime; preserve None. - assert "async def optional_param(x: int | None = ...) -> bool:" in stubs + assert "async def optional_param(*, x: int | None = ...) -> bool:" in stubs # Multi-arm union with None. - assert "async def union_param(x: int | str | None) -> str:" in stubs + assert "async def union_param(*, x: int | str | None) -> str:" in stubs def test_generate_type_stubs_skips_non_identifier_tool_names() -> None: @@ -1778,7 +1843,7 @@ def normal(x: int) -> int: "normal": normal, }) - assert "async def normal(x: int) -> int:" in stubs + assert "async def normal(*, x: int) -> int:" in stubs assert "weird-name" not in stubs assert "injected" not in stubs assert "async def async(" not in stubs diff --git a/python/packages/monty/tests/monty/test_monty_runtime.py b/python/packages/monty/tests/monty/test_monty_runtime.py index 52654a3d584..a166a2a7ec9 100644 --- a/python/packages/monty/tests/monty/test_monty_runtime.py +++ b/python/packages/monty/tests/monty/test_monty_runtime.py @@ -7,9 +7,11 @@ from typing import Any import pytest +from agent_framework import FunctionTool from pydantic_monty import MontyRuntimeError from agent_framework_monty import MontyExecuteCodeTool +from agent_framework_monty._execute_code_tool import _make_tool_callback from agent_framework_monty._monty_bridge import InlineCodeBridge @@ -60,3 +62,81 @@ async def test_legacy_and_feed_duration_limits_are_rejected_as_ambiguous(legacy_ with pytest.raises(ValueError, match="Specify only one of max_duration_secs and max_feed_duration_secs"): await bridge.run("print('must not run')") + + +@pytest.mark.parametrize("recording_host_tool", ["positional_or_keyword", "keyword_only"], indirect=True) +async def test_positional_tool_call_is_rejected_before_any_host_tool_runs( + recording_host_tool: tuple[FunctionTool, list[str | None]], +) -> None: + host_tool, calls = recording_host_tool + execute_code = MontyExecuteCodeTool(tools=[host_tool], include_detailed_errors=True) + + contents = await execute_code._run_code(code='await record(label="first")\nawait record("second")') + + assert [content.type for content in contents] == ["error"] + assert "positional" in (contents[0].error_details or "").lower() + # Type checking fails before execution, so even the valid first call never reaches the host. + assert calls == [] + + +@pytest.mark.parametrize("recording_host_tool", ["positional_or_keyword", "keyword_only"], indirect=True) +async def test_positional_tool_call_is_rejected_when_type_checking_is_bypassed( + recording_host_tool: tuple[FunctionTool, list[str | None]], +) -> None: + host_tool, calls = recording_host_tool + bridge = InlineCodeBridge({"record": _make_tool_callback(host_tool)}) + + with pytest.raises(MontyRuntimeError, match="record accepts keyword arguments only"): + await bridge.run('await record("second")') + + assert calls == [] + + +@pytest.mark.parametrize("recording_host_tool", ["positional_or_keyword", "keyword_only"], indirect=True) +async def test_keyword_tool_call_runs_and_omitted_parameters_use_defaults( + recording_host_tool: tuple[FunctionTool, list[str | None]], +) -> None: + host_tool, calls = recording_host_tool + execute_code = MontyExecuteCodeTool(tools=[host_tool]) + + contents = await execute_code._run_code(code='await record(label="named")\nawait record()') + + assert not any(content.type == "error" for content in contents) + assert calls == ["named", None] + + +@pytest.mark.parametrize( + ("unsupplied_parameter_host_tool", "expected_call"), + [ + ("context", "x:FunctionInvocationContext"), + ("var_keyword", "x"), + ("var_positional", "x"), + ("method", "x"), + ], + indirect=["unsupplied_parameter_host_tool"], +) +async def test_typed_call_does_not_require_parameters_that_guest_code_cannot_supply( + unsupplied_parameter_host_tool: tuple[FunctionTool, list[str]], + expected_call: str, +) -> None: + host_tool, calls = unsupplied_parameter_host_tool + execute_code = MontyExecuteCodeTool(tools=[host_tool], include_detailed_errors=True) + + contents = await execute_code._run_code(code='await lookup(query="x")') + + assert not any(content.type == "error" for content in contents) + assert calls == [expected_call] + + +@pytest.mark.parametrize("unsupplied_parameter_host_tool", ["var_keyword"], indirect=True) +async def test_typed_call_with_undeclared_keyword_is_rejected_before_any_host_tool_runs( + unsupplied_parameter_host_tool: tuple[FunctionTool, list[str]], +) -> None: + host_tool, calls = unsupplied_parameter_host_tool + execute_code = MontyExecuteCodeTool(tools=[host_tool], include_detailed_errors=True) + + contents = await execute_code._run_code(code='await lookup(query="x", extra=1)') + + assert [content.type for content in contents] == ["error"] + assert "unknown-argument" in (contents[0].error_details or "") + assert calls == []