From 5360db5873f72b9185211961e746bdc7c9e2627d Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 6 Nov 2026 10:33:01 -0500 Subject: [PATCH 1/6] fix(hyperlight): interrupt Python guest execution and enforce time limits Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3d6807e7-70a6-4c43-8dc1-7bab20376cd4 --- python/packages/hyperlight/README.md | 35 ++++ .../_execute_code_tool.py | 149 ++++++++++++++++-- .../agent_framework_hyperlight/_provider.py | 5 + .../hyperlight/tests/hyperlight/conftest.py | 64 +++++++- .../hyperlight/test_execution_control.py | 67 ++++++++ .../tests/hyperlight/test_execution_limits.py | 144 +++++++++++++++++ .../hyperlight/test_hyperlight_codeact.py | 32 +++- .../references/integrations.md | 8 + 8 files changed, 491 insertions(+), 13 deletions(-) create mode 100644 python/packages/hyperlight/tests/hyperlight/test_execution_control.py create mode 100644 python/packages/hyperlight/tests/hyperlight/test_execution_limits.py diff --git a/python/packages/hyperlight/README.md b/python/packages/hyperlight/README.md index 13abf69f832..216de3d19ee 100644 --- a/python/packages/hyperlight/README.md +++ b/python/packages/hyperlight/README.md @@ -38,6 +38,41 @@ Initialization, snapshot restore, and execution failures follow the same policy. Failed sandbox entries are retired; cancellation and system-exit signals still propagate without becoming ordinary execution errors. +## Execution limits + +Guest execution has a finite 30-second budget by default. Both +`HyperlightCodeActProvider` and `HyperlightExecuteCodeTool` accept the host-side +`execution_timeout_seconds` option, which must be a positive finite number. +Applications that intentionally run longer must configure a larger budget. +The option is captured in provider run snapshots and serialized state; it is +not a model-controlled `execute_code` argument. + +Reaching the budget returns an error content with +`Code execution exceeded its time limit.` Caller task cancellation interrupts +guest execution and propagates `asyncio.CancelledError`. Cancellation waits for +native execution and its interruption thread to finish, even if the caller +cancels again. Queued cancellation does not interrupt another invocation. +The next invocation restores the warm snapshot before reusing the sandbox. + +The budget starts on the sandbox's owning worker, not while waiting for that +worker. Initialization and warm-up share the invocation's budget, but runtime +initialization and host tool callbacks cannot be interrupted. Host tools must +enforce their own cancellation and time limits. Completed results and unrelated +execution failures are not replaced with a timeout that arrives late. + +This behavior requires an SDK with the cross-thread `Sandbox.interrupt_handle()` +API from [hyperlight-dev/hyperlight-sandbox#275](https://github.com/hyperlight-dev/hyperlight-sandbox/pull/275). +The currently declared released dependency does not yet include this API; +a compatible SDK release and dependency-floor update are required before shipping. + +To run the native regressions on a KVM host with the linked SDK and both +backends installed: + +```bash +HYPERLIGHT_RUN_GUEST_TESTS=true uv run --directory python/packages/hyperlight pytest \ + tests/hyperlight/test_execution_limits.py -m integration +``` + ## Installation ```bash diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py index 645e0f2de45..673f71a3692 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py @@ -5,6 +5,7 @@ import asyncio import contextvars import logging +import math import mimetypes import os import shutil @@ -57,6 +58,10 @@ OUTPUT_TRAVERSAL_ENTRIES_PER_FILE = 10 OUTPUT_TRAVERSAL_MAX_ENTRIES = 10_000 OUTPUT_TRAVERSAL_MAX_DEPTH = 32 +DEFAULT_EXECUTION_TIMEOUT_SECONDS = 30.0 +_execution_cancellation: contextvars.ContextVar[threading.Event | None] = contextvars.ContextVar( + "hyperlight_execution_cancellation", default=None +) EXECUTE_CODE_INPUT_SCHEMA: dict[str, Any] = { "type": "object", @@ -80,6 +85,68 @@ class _OutputCleanupError(RuntimeError): """Raised when sandbox output cannot be safely cleaned within configured bounds.""" +class _GuestExecutionTimeout(TimeoutError): + """Raised when the invocation's guest execution budget expires.""" + + +class _ExecutionControl: + def __init__(self, timeout_seconds: float) -> None: + self._timeout_seconds = timeout_seconds + self._deadline: float | None = None + self._cancelled = _execution_cancellation.get() or threading.Event() + + def start(self) -> float: + if self._deadline is None: + self._deadline = time.monotonic() + self._timeout_seconds + self.check() + return self._deadline + + def check(self) -> None: + if self._cancelled.is_set(): + raise asyncio.CancelledError + if self._deadline is not None and time.monotonic() >= self._deadline: + raise _GuestExecutionTimeout("Code execution exceeded its time limit.") + + def run(self, sandbox: Any, code: str) -> Any: + deadline = self.start() + handle = sandbox.interrupt_handle() + self.check() + completed = threading.Event() + timed_out = threading.Event() + interruption_errors: list[str] = [] + + def _interrupt() -> None: + try: + while not completed.is_set(): + if time.monotonic() >= deadline: + timed_out.set() + if (self._cancelled.is_set() or timed_out.is_set()) and handle.kill(): + return + completed.wait(0.01) + except Exception as exc: + interruption_errors.append(str(exc)) + + # Only the independently owned handle crosses threads, never the sandbox. + interruption = threading.Thread(target=_interrupt, name="hl-interrupt") + interruption.start() + try: + self.check() + try: + result = sandbox.run(code=code) + except InterruptedError: + if self._cancelled.is_set(): + raise asyncio.CancelledError from None + if timed_out.is_set(): + raise _GuestExecutionTimeout("Code execution exceeded its time limit.") from None + raise + finally: + completed.set() + interruption.join() + if interruption_errors: + raise RuntimeError("Could not interrupt Hyperlight guest execution.") + return result + + def _output_cleanup_error_content(error: _OutputCleanupError) -> Content: return Content.from_error(message="Execution error", error_details=str(error)) @@ -113,6 +180,7 @@ class _RunConfig: max_output_file_bytes: int = DEFAULT_MAX_OUTPUT_FILE_BYTES max_output_total_bytes: int = DEFAULT_MAX_OUTPUT_TOTAL_BYTES include_detailed_errors: bool = False + execution_timeout_seconds: float = DEFAULT_EXECUTION_TIMEOUT_SECONDS @property def mounted_paths(self) -> tuple[str, ...]: @@ -203,7 +271,8 @@ def _wrapped() -> tuple[bool, Any]: try: return True, fn() except BaseException as exc: - self._reusable = False + if not isinstance(exc, (asyncio.CancelledError, _GuestExecutionTimeout)): + self._reusable = False exc_type: type[BaseException] exc_args: tuple[str, ...] # Capture args (usually (message,)) so the re-raised exception keeps the @@ -213,7 +282,11 @@ def _wrapped() -> tuple[bool, Any]: # unsendable object in args, forwarding it as-is would re-introduce the # same cross-thread Drop hazard the traceback nulling avoids. Strings # are always sendable. Fall back to the str() form if args is empty. - if isinstance(exc, Exception) and not include_detailed_errors: + if ( + isinstance(exc, Exception) + and not include_detailed_errors + and not isinstance(exc, _GuestExecutionTimeout) + ): exc_type = RuntimeError exc_args = ("Code execution failed.",) else: @@ -267,6 +340,7 @@ def execute( max_output_files: int, max_output_file_bytes: int, max_output_total_bytes: int, + control: _ExecutionControl, include_detailed_errors: bool = False, ) -> list[Content]: """Restore + run + build sendable contents — all on the worker thread. @@ -277,6 +351,7 @@ def execute( cleanup_max_entries = _output_traversal_entry_limit(max_output_files) def _on_worker() -> list[Content]: + control.start() if not self._reusable: return [_output_cleanup_error_content(_OutputCleanupError("Could not clear sandbox output safely."))] @@ -295,7 +370,7 @@ def _on_worker() -> list[Content]: contents: list[Content] | None = None try: - result = sandbox.run(code=code) + result = control.run(sandbox, code) try: contents = build_contents( result=result, @@ -1426,8 +1501,10 @@ def execute(self, *, config: _RunConfig, code: str) -> list[Content]: Entries whose output cannot be cleaned safely are evicted before another invocation. """ cache_key = config.cache_key() - entry = self._get_or_create_entry(config) + control = _ExecutionControl(config.execution_timeout_seconds) + entry: _SandboxEntry | None = None try: + entry = self._get_or_create_entry(config, control=control) return entry.worker.execute( code=code, output_dir=entry.output_dir, @@ -1435,10 +1512,15 @@ def execute(self, *, config: _RunConfig, code: str) -> list[Content]: max_output_files=config.max_output_files, max_output_file_bytes=config.max_output_file_bytes, max_output_total_bytes=config.max_output_total_bytes, + control=control, include_detailed_errors=config.include_detailed_errors, ) + except _GuestExecutionTimeout: + return [ + Content.from_error(message="Execution error", error_details="Code execution exceeded its time limit.") + ] finally: - if not entry.worker.is_reusable(): + if entry is not None and not entry.worker.is_reusable(): self._discard_entry(cache_key=cache_key, entry=entry) def _discard_entry(self, *, cache_key: tuple[Any, ...], entry: _SandboxEntry) -> None: @@ -1449,12 +1531,14 @@ def _discard_entry(self, *, cache_key: tuple[Any, ...], entry: _SandboxEntry) -> entry.worker.dispose() self._cleanup_executor.submit(entry.cleanup_temp_dirs) - def _get_or_create_entry(self, config: _RunConfig) -> _SandboxEntry: + def _get_or_create_entry(self, config: _RunConfig, *, control: _ExecutionControl | None = None) -> _SandboxEntry: cache_key = config.cache_key() with self._entries_lock: entry = self._entries.get(cache_key) if entry is None: - entry = self._create_entry(config) + entry = self._create_entry( + config, control=control or _ExecutionControl(config.execution_timeout_seconds) + ) self._entries[cache_key] = entry return entry @@ -1476,7 +1560,7 @@ def close(self) -> None: # were already moved into the per-worker disposal closure inside dispose(). del entries - def _create_entry(self, config: _RunConfig) -> _SandboxEntry: + def _create_entry(self, config: _RunConfig, *, control: _ExecutionControl) -> _SandboxEntry: input_dir_handle = TemporaryDirectory() if config.filesystem_enabled else None output_dir_handle = TemporaryDirectory() if config.filesystem_enabled else None @@ -1521,18 +1605,19 @@ def _configure_sandbox(*, sandbox: Any, expand_missing_scheme: bool) -> None: ) def _build_sandbox() -> tuple[Any, Any]: + control.start() sandbox = _create_sandbox() _configure_sandbox(sandbox=sandbox, expand_missing_scheme=False) try: - sandbox.run("None") + control.run(sandbox, "None") except RuntimeError as exc: if not _should_retry_allowed_domain_registration(error=exc, allowed_domains=config.allowed_domains): raise sandbox = _create_sandbox() _configure_sandbox(sandbox=sandbox, expand_missing_scheme=True) - sandbox.run("None") + control.run(sandbox, "None") snapshot = sandbox.snapshot() return sandbox, snapshot @@ -1596,6 +1681,9 @@ class HyperlightExecuteCodeTool(FunctionTool): Mappings are copied, including entries for tools registered later. include_detailed_errors: Expose host exception and execution diagnostics only for a trusted channel. Defaults to ``False`` and is independent of outer-client error/logging settings. + execution_timeout_seconds: Positive finite guest execution budget, in seconds. Defaults to 30. + Cancellation and timeout wait for native execution to stop. Runtime initialization and + host tool callbacks cannot be interrupted; host tools must enforce their own limits. """ def __init__( @@ -1611,6 +1699,7 @@ def __init__( max_output_files: int = DEFAULT_MAX_OUTPUT_FILES, max_output_file_bytes: int = DEFAULT_MAX_OUTPUT_FILE_BYTES, max_output_total_bytes: int = DEFAULT_MAX_OUTPUT_TOTAL_BYTES, + execution_timeout_seconds: float = DEFAULT_EXECUTION_TIMEOUT_SECONDS, backend: str = DEFAULT_HYPERLIGHT_BACKEND, module: str | None = DEFAULT_HYPERLIGHT_MODULE, module_path: str | None = None, @@ -1619,6 +1708,10 @@ def __init__( ) -> None: if not isinstance(include_detailed_errors, bool): raise TypeError("include_detailed_errors must be a bool.") + if isinstance(execution_timeout_seconds, bool) or not isinstance(execution_timeout_seconds, (int, float)): + raise TypeError("execution_timeout_seconds must be a number.") + if not math.isfinite(execution_timeout_seconds) or execution_timeout_seconds <= 0: + raise ValueError("execution_timeout_seconds must be positive and finite.") normalized_description_format = _normalize_tool_description_format(tool_description_format) max_output_files = _validate_positive_integer(name="max_output_files", value=max_output_files) max_output_file_bytes = _validate_positive_integer(name="max_output_file_bytes", value=max_output_file_bytes) @@ -1635,6 +1728,7 @@ def __init__( self._registry = _registry or _SandboxRegistry() self._default_approval_mode: ApprovalMode = approval_mode or "never_require" self._include_detailed_errors = include_detailed_errors + self._execution_timeout_seconds = execution_timeout_seconds self._workspace_root = _resolve_workspace_root(workspace_root) self._backend: str = backend self._module: str | None = module @@ -1819,6 +1913,7 @@ def create_run_tool(self) -> HyperlightExecuteCodeTool: max_output_files=self._max_output_files, max_output_file_bytes=self._max_output_file_bytes, max_output_total_bytes=self._max_output_total_bytes, + execution_timeout_seconds=self._execution_timeout_seconds, backend=self._backend, module=self._module, module_path=self._module_path, @@ -1834,6 +1929,7 @@ def build_serializable_state(self) -> dict[str, Any]: "module_path": config.module_path, "approval_mode": config.approval_mode, "include_detailed_errors": config.include_detailed_errors, + "execution_timeout_seconds": config.execution_timeout_seconds, "max_output_files": config.max_output_files, "max_output_file_bytes": config.max_output_file_bytes, "max_output_total_bytes": config.max_output_total_bytes, @@ -1911,13 +2007,44 @@ def _build_run_config(self) -> _RunConfig: max_output_file_bytes=max_output_file_bytes, max_output_total_bytes=max_output_total_bytes, include_detailed_errors=include_detailed_errors, + execution_timeout_seconds=self._execution_timeout_seconds, ) async def _run_code(self, *, code: str) -> list[Content]: include_detailed_errors = self._include_detailed_errors try: config = self._build_run_config() - return await asyncio.to_thread(self._registry.execute, config=config, code=code) + cancelled = threading.Event() + token = _execution_cancellation.set(cancelled) + try: + current_context = contextvars.copy_context() + finally: + _execution_cancellation.reset(token) + execution = asyncio.get_running_loop().run_in_executor( + None, lambda: current_context.run(self._registry.execute, config=config, code=code) + ) + try: + return await asyncio.shield(execution) + except asyncio.CancelledError: + cancelled.set() + # Shield the owning worker until native execution and its interruption + # thread have finished, including when the caller cancels again. + while not execution.done(): + try: + await asyncio.shield(execution) + except asyncio.CancelledError: + continue + except Exception: + break + try: + execution.result() + except asyncio.CancelledError: + pass + except Exception: + logger.warning( + "Hyperlight execution failed during cancellation cleanup.", exc_info=include_detailed_errors + ) + raise except Exception: if include_detailed_errors: raise diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_provider.py b/python/packages/hyperlight/agent_framework_hyperlight/_provider.py index a72fd032f44..bfb76daf88a 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_provider.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_provider.py @@ -11,6 +11,7 @@ from agent_framework._tools import ApprovalMode, _ToolDescriptionFormat # pyright: ignore[reportPrivateUsage] from ._execute_code_tool import ( + DEFAULT_EXECUTION_TIMEOUT_SECONDS, DEFAULT_MAX_OUTPUT_FILE_BYTES, DEFAULT_MAX_OUTPUT_FILES, DEFAULT_MAX_OUTPUT_TOTAL_BYTES, @@ -31,6 +32,8 @@ class HyperlightCodeActProvider(ContextProvider): Mappings are copied, including entries for tools registered later. include_detailed_errors: Expose host exception and execution diagnostics only for a trusted channel. Defaults to ``False`` and is independent of outer-client error/logging settings. + execution_timeout_seconds: Positive finite guest execution budget, in seconds. Defaults to 30. + Runtime initialization and host tool callbacks cannot be interrupted. """ DEFAULT_SOURCE_ID = "hyperlight_codeact" @@ -49,6 +52,7 @@ def __init__( max_output_files: int = DEFAULT_MAX_OUTPUT_FILES, max_output_file_bytes: int = DEFAULT_MAX_OUTPUT_FILE_BYTES, max_output_total_bytes: int = DEFAULT_MAX_OUTPUT_TOTAL_BYTES, + execution_timeout_seconds: float = DEFAULT_EXECUTION_TIMEOUT_SECONDS, backend: str = "wasm", module: str | None = "python_guest.path", module_path: str | None = None, @@ -66,6 +70,7 @@ def __init__( max_output_files=max_output_files, max_output_file_bytes=max_output_file_bytes, max_output_total_bytes=max_output_total_bytes, + execution_timeout_seconds=execution_timeout_seconds, backend=backend, module=module, module_path=module_path, diff --git a/python/packages/hyperlight/tests/hyperlight/conftest.py b/python/packages/hyperlight/tests/hyperlight/conftest.py index 23d4bfa8515..21db573f11b 100644 --- a/python/packages/hyperlight/tests/hyperlight/conftest.py +++ b/python/packages/hyperlight/tests/hyperlight/conftest.py @@ -6,11 +6,70 @@ import threading from collections.abc import Callable from types import SimpleNamespace -from typing import Any +from typing import Any, cast import pytest from agent_framework import FunctionTool +from agent_framework_hyperlight import HyperlightExecuteCodeTool +from agent_framework_hyperlight import _execute_code_tool as execute_code_module + + +@pytest.fixture +def interruptible_execution(monkeypatch: pytest.MonkeyPatch): + entered = threading.Event() + release = threading.Event() + interrupted = threading.Event() + kill_attempted = threading.Event() + completed = threading.Event() + + class Sandbox: + def __init__(self, **kwargs: Any) -> None: + self._host_callback = False + + def interrupt_handle(self) -> SimpleNamespace: + def kill() -> bool: + kill_attempted.set() + interrupted.set() + return not self._host_callback + + return SimpleNamespace(kill=kill) + + def run(self, code: str) -> SimpleNamespace: + if code in {"guest", "host"}: + self._host_callback = code == "host" + entered.set() + try: + if self._host_callback: + if not release.wait(5): + raise RuntimeError("Host callback was not released.") + elif not interrupted.wait(5): + raise RuntimeError("Guest was not interrupted.") + if interrupted.is_set(): + raise InterruptedError("Guest execution interrupted.") + finally: + completed.set() + self._host_callback = False + return SimpleNamespace(success=True, stdout="ok\n", stderr="") + + def snapshot(self) -> object: + return object() + + def restore(self, snapshot: object) -> None: + interrupted.clear() + + monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: Sandbox) + execute_code = HyperlightExecuteCodeTool(execution_timeout_seconds=0.1) + yield SimpleNamespace( + tool=execute_code, + entered=entered, + release=release, + kill_attempted=kill_attempted, + completed=completed, + ) + release.set() + cast(Any, execute_code._registry).close() + @pytest.fixture def host_error_detail() -> str: @@ -70,6 +129,9 @@ def snapshot(self) -> object: self._record("snapshot") return object() + def interrupt_handle(self) -> SimpleNamespace: + return SimpleNamespace(kill=lambda: False) + def restore(self, snapshot: object) -> None: self._record("restore") diff --git a/python/packages/hyperlight/tests/hyperlight/test_execution_control.py b/python/packages/hyperlight/tests/hyperlight/test_execution_control.py new file mode 100644 index 00000000000..9118c97e732 --- /dev/null +++ b/python/packages/hyperlight/tests/hyperlight/test_execution_control.py @@ -0,0 +1,67 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import asyncio + + +async def test_timeout_returns_error_and_allows_reuse(interruptible_execution) -> None: + native = interruptible_execution + result = await native.tool.invoke(arguments={"code": "guest"}) + + assert native.completed.is_set() + assert len(result) == 1 + assert result[0].type == "error" + assert result[0].error_details == "Code execution exceeded its time limit." + recovered = await native.tool.invoke(arguments={"code": "None"}) + assert recovered[0].text == "ok\n" + + +async def test_repeated_cancellation_waits_for_host_callback_to_finish(interruptible_execution) -> None: + native = interruptible_execution + execution = asyncio.create_task(native.tool.invoke(arguments={"code": "host"})) + try: + assert await asyncio.to_thread(native.entered.wait, 2) + execution.cancel() + assert await asyncio.to_thread(native.kill_attempted.wait, 2) + execution.cancel() + await asyncio.sleep(0) + assert not execution.done() + assert not native.completed.is_set() + finally: + native.release.set() + try: + await asyncio.wait_for(execution, 2) + except asyncio.CancelledError: + pass + else: + raise AssertionError("Caller cancellation was not preserved.") + assert native.completed.is_set() + recovered = await native.tool.invoke(arguments={"code": "None"}) + assert recovered[0].text == "ok\n" + + +async def test_queued_cancellation_does_not_interrupt_another_invocation(interruptible_execution) -> None: + native = interruptible_execution + native.tool._execution_timeout_seconds = 5 + active = asyncio.create_task(native.tool.invoke(arguments={"code": "host"})) + queued = None + try: + assert await asyncio.to_thread(native.entered.wait, 2) + queued = asyncio.create_task(native.tool.invoke(arguments={"code": "None"})) + await asyncio.sleep(0.02) + queued.cancel() + await asyncio.sleep(0) + assert not native.kill_attempted.is_set() + finally: + native.release.set() + result = await asyncio.wait_for(active, 2) + assert result[0].text == "ok\n" + assert queued is not None + try: + await asyncio.wait_for(queued, 2) + except asyncio.CancelledError: + pass + else: + raise AssertionError("Queued cancellation was not preserved.") + assert not native.kill_attempted.is_set() diff --git a/python/packages/hyperlight/tests/hyperlight/test_execution_limits.py b/python/packages/hyperlight/tests/hyperlight/test_execution_limits.py new file mode 100644 index 00000000000..d1dc4c80c50 --- /dev/null +++ b/python/packages/hyperlight/tests/hyperlight/test_execution_limits.py @@ -0,0 +1,144 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import asyncio +import os +import time +from typing import Any, cast + +import pytest +from agent_framework import tool + +from agent_framework_hyperlight import HyperlightExecuteCodeTool + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + os.environ.get("HYPERLIGHT_RUN_GUEST_TESTS") != "true", + reason="Requires explicit Hyperlight KVM guest-test configuration.", + ), +] + + +@pytest.mark.parametrize("backend", ["wasm", "hyperlight-js"]) +async def test_cancellation_stops_guest_before_reuse(backend: str) -> None: + loop = asyncio.get_running_loop() + entered = asyncio.Event() + completed = asyncio.Event() + deadline = 0.0 + + @tool + def started() -> None: + """Signal guest entry.""" + nonlocal deadline + deadline = time.monotonic() + 5 + loop.call_soon_threadsafe(entered.set) + + @tool + def keep_running() -> bool: + """Bound the baseline workload using the host clock.""" + return time.monotonic() < deadline + + @tool + def finished() -> None: + """Signal guest completion.""" + loop.call_soon_threadsafe(completed.set) + + execute_code = HyperlightExecuteCodeTool( + tools=[started, keep_running, finished], + backend=backend, + module=None if backend == "hyperlight-js" else "python_guest.path", + ) + code = ( + "call_tool('started')\nwhile call_tool('keep_running'):\n pass\ncall_tool('finished')" + if backend == "wasm" + else "call_tool('started'); while (call_tool('keep_running')) {} call_tool('finished');" + ) + try: + warmup = await execute_code.invoke(arguments={"code": "print('ready')" if backend == "wasm" else "1;"}) + assert not any(item.type == "error" for item in warmup), warmup + execution = asyncio.create_task(execute_code.invoke(arguments={"code": code})) + await asyncio.wait_for(entered.wait(), 10) + start = time.monotonic() + execution.cancel() + with pytest.raises(asyncio.CancelledError): + await execution + recovered = await asyncio.wait_for( + execute_code.invoke( + arguments={"code": "print('recovered')" if backend == "wasm" else "console.log('recovered');"} + ), + 10, + ) + assert not any(item.type == "error" for item in recovered), recovered + assert not completed.is_set(), "Guest continued executing after caller cancellation." + assert time.monotonic() - start < 3, "Cancellation did not promptly release guest execution." + assert any(item.type == "text" and "recovered" in (item.text or "") for item in recovered) + finally: + cast(Any, execute_code._registry).close() + + +@pytest.mark.parametrize("backend", ["wasm", "hyperlight-js"]) +async def test_configured_timeout_interrupts_cpu_work_and_allows_reuse(backend: str) -> None: + warm_tool = HyperlightExecuteCodeTool( + backend=backend, module=None if backend == "hyperlight-js" else "python_guest.path" + ) + limited_tool = HyperlightExecuteCodeTool( + backend=backend, + module=None if backend == "hyperlight-js" else "python_guest.path", + execution_timeout_seconds=0.1, + _registry=warm_tool._registry, + ) + try: + warmup = await warm_tool.invoke(arguments={"code": "print('ready')" if backend == "wasm" else "1;"}) + assert not any(item.type == "error" for item in warmup), warmup + start = time.monotonic() + result = await limited_tool.invoke( + arguments={ + "code": ( + "for value in range(100_000_000):\n value * value" + if backend == "wasm" + else "for (let value = 0; value < 100000000; value++) { value * value; }" + ) + } + ) + assert time.monotonic() - start < 3 + errors = [item for item in result if item.type == "error"] + assert len(errors) == 1 + assert errors[0].error_details == "Code execution exceeded its time limit." + recovered = await asyncio.wait_for( + warm_tool.invoke( + arguments={"code": "print('recovered')" if backend == "wasm" else "console.log('recovered');"} + ), + 10, + ) + assert not any(item.type == "error" for item in recovered), recovered + assert any(item.type == "text" and "recovered" in (item.text or "") for item in recovered) + finally: + cast(Any, warm_tool._registry).close() + + +async def test_default_timeout_stops_guest_before_reuse() -> None: + deadline = 0.0 + + @tool + def keep_running() -> bool: + """Bound the baseline workload using the host clock.""" + return time.monotonic() < deadline + + execute_code = HyperlightExecuteCodeTool(tools=[keep_running]) + try: + warmup = await execute_code.invoke(arguments={"code": "print('ready')"}) + assert not any(item.type == "error" for item in warmup), warmup + deadline = time.monotonic() + 32 + result = await execute_code.invoke( + arguments={"code": "while call_tool('keep_running'):\n pass\nprint('finished')"} + ) + errors = [item for item in result if item.type == "error"] + assert len(errors) == 1, "The default execution limit was not enforced." + assert errors[0].error_details == "Code execution exceeded its time limit." + recovered = await asyncio.wait_for(execute_code.invoke(arguments={"code": "print('recovered')"}), 10) + assert any(item.type == "text" and "recovered" in (item.text or "") for item in recovered) + assert not any(item.type == "error" for item in recovered), recovered + finally: + cast(Any, execute_code._registry).close() diff --git a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py index d7ee0381d60..80f4da03aad 100644 --- a/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py +++ b/python/packages/hyperlight/tests/hyperlight/test_hyperlight_codeact.py @@ -21,7 +21,7 @@ from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory -from types import MappingProxyType +from types import MappingProxyType, SimpleNamespace from typing import Any, Literal, cast from unittest.mock import DEFAULT, patch @@ -325,6 +325,9 @@ def run(self, code: str) -> _FakeResult: def snapshot(self) -> str: return "snapshot" + def interrupt_handle(self) -> SimpleNamespace: + return SimpleNamespace(kill=lambda: False) + def restore(self, snapshot: Any) -> None: self.restore_calls.append(snapshot) @@ -2504,6 +2507,9 @@ def run(self, code: str) -> _FakeResult: def snapshot(self) -> str: return "snapshot" + def interrupt_handle(self) -> SimpleNamespace: + return SimpleNamespace(kill=lambda: False) + def restore(self, snapshot: Any) -> None: del snapshot @@ -2611,6 +2617,30 @@ async def test_provider_forwards_output_limits_to_run_tool_and_serializable_stat json.dumps(state) +@pytest.mark.parametrize("constructor", [HyperlightExecuteCodeTool, HyperlightCodeActProvider]) +@pytest.mark.parametrize("value", [True, "30", None, 0, -1, float("inf"), float("-inf"), float("nan")]) +def test_execution_timeout_requires_a_positive_finite_number(constructor: Any, value: Any) -> None: + with pytest.raises((TypeError, ValueError), match="execution_timeout_seconds"): + constructor(execution_timeout_seconds=value) + + +async def test_provider_captures_execution_timeout_without_changing_sandbox_identity() -> None: + runtime = _FakeRuntime() + provider = HyperlightCodeActProvider(execution_timeout_seconds=0.25, _registry=runtime) + context = _FakeSessionContext() + state: dict[str, Any] = {} + + await provider.before_run(agent=object(), session=None, context=cast(Any, context), state=state) + run_tool = context.tools[0][1][0] + assert isinstance(run_tool, HyperlightExecuteCodeTool) + await run_tool.invoke(arguments={"code": "None"}) + + assert runtime.calls[0][0].execution_timeout_seconds == 0.25 + assert state[provider.source_id]["execution_timeout_seconds"] == 0.25 + default_config = HyperlightExecuteCodeTool(_registry=runtime)._build_run_config() + assert runtime.calls[0][0].cache_key() == default_config.cache_key() + + async def test_output_limits_are_isolated_when_registry_is_shared( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/skills/agent-framework-builder-python/references/integrations.md b/skills/agent-framework-builder-python/references/integrations.md index c19bb242bb4..3a9f8570dba 100644 --- a/skills/agent-framework-builder-python/references/integrations.md +++ b/skills/agent-framework-builder-python/references/integrations.md @@ -63,6 +63,14 @@ Monty and Hyperlight redact host tool exception diagnostics by default. Enable is separate from the outer chat client's error policy and sensitive-data logging; outer redaction cannot repair a diagnostic already caught and returned by generated code. +Hyperlight guest execution defaults to a 30-second budget. Configure the positive finite +host-side `execution_timeout_seconds` option on `HyperlightCodeActProvider` or +`HyperlightExecuteCodeTool` for longer work. Caller task cancellation interrupts guest execution +and waits for the owning worker to finish before propagating. Neither cancellation nor the budget +can interrupt runtime initialization or host tool callbacks; enforce limits inside host tools too. +This requires a compatible SDK with `Sandbox.interrupt_handle()`; verify the selected package +release supports it before relying on these limits. + For Monty, distinguish interpreter `resource_limits` from host-side attachment capture. Capture has per-call file-count and raw-byte limits shared across read-write mounts. Omitted attachments do not undo filesystem writes. Monty requires `pydantic-monty>=1.0`. Use `max_feed_duration_secs` for the total sandbox execution budget or From 53f6526768b1605e8598236422515fb2a01ec7ac Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Mon, 9 Nov 2026 12:16:15 -0500 Subject: [PATCH 2/6] chore: updates hyperlight package to required version --- python/packages/hyperlight/README.md | 8 ++--- python/packages/hyperlight/pyproject.toml | 6 ++-- python/uv.lock | 36 +++++++++++------------ 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/python/packages/hyperlight/README.md b/python/packages/hyperlight/README.md index 216de3d19ee..f6c2333a30d 100644 --- a/python/packages/hyperlight/README.md +++ b/python/packages/hyperlight/README.md @@ -60,12 +60,10 @@ initialization and host tool callbacks cannot be interrupted. Host tools must enforce their own cancellation and time limits. Completed results and unrelated execution failures are not replaced with a timeout that arrives late. -This behavior requires an SDK with the cross-thread `Sandbox.interrupt_handle()` -API from [hyperlight-dev/hyperlight-sandbox#275](https://github.com/hyperlight-dev/hyperlight-sandbox/pull/275). -The currently declared released dependency does not yet include this API; -a compatible SDK release and dependency-floor update are required before shipping. +This behavior uses the cross-thread `Sandbox.interrupt_handle()` API included +in the required Hyperlight SDK 0.9.0 or later. -To run the native regressions on a KVM host with the linked SDK and both +To run the native regressions on a KVM host with the required SDK and both backends installed: ```bash diff --git a/python/packages/hyperlight/pyproject.toml b/python/packages/hyperlight/pyproject.toml index 8cd17c2d765..63694e024ce 100644 --- a/python/packages/hyperlight/pyproject.toml +++ b/python/packages/hyperlight/pyproject.toml @@ -24,9 +24,9 @@ classifiers = [ ] dependencies = [ "agent-framework-core>=1.19.0,<2", - "hyperlight-sandbox>=0.8.0,<0.9", - "hyperlight-sandbox-backend-wasm>=0.8.0,<0.9 ; (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'win32' and platform_machine == 'AMD64')", - "hyperlight-sandbox-python-guest>=0.8.0,<0.9", + "hyperlight-sandbox>=0.9.0,<0.10", + "hyperlight-sandbox-backend-wasm>=0.9.0,<0.10 ; (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'win32' and platform_machine == 'AMD64')", + "hyperlight-sandbox-python-guest>=0.9.0,<0.10", ] [tool.uv] diff --git a/python/uv.lock b/python/uv.lock index 732b31233d2..eb88563d099 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -898,9 +898,9 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "agent-framework-core", editable = "packages/core" }, - { name = "hyperlight-sandbox", specifier = ">=0.8.0,<0.9" }, - { name = "hyperlight-sandbox-backend-wasm", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')", specifier = ">=0.8.0,<0.9" }, - { name = "hyperlight-sandbox-python-guest", specifier = ">=0.8.0,<0.9" }, + { name = "hyperlight-sandbox", specifier = ">=0.9.0,<0.10" }, + { name = "hyperlight-sandbox-backend-wasm", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')", specifier = ">=0.9.0,<0.10" }, + { name = "hyperlight-sandbox-python-guest", specifier = ">=0.9.0,<0.10" }, ] [[package]] @@ -2780,35 +2780,35 @@ wheels = [ [[package]] name = "hyperlight-sandbox" -version = "0.8.0" +version = "0.9.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/ee/3d97cd3ca57addeae74eaf82a1d64475f151f884e4bb8dca874b01ba3788/hyperlight_sandbox-0.8.0.tar.gz", hash = "sha256:3a7c9f4a3e3c41ddd8bd2b9f48db6ece652b2c18be384f459d6b168d33c063ee", size = 11861, upload-time = "2026-10-17T17:19:57.639Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/06/3b3e54583ea790bd154dcec18a7eb73394dce50337bff99fb4513991d443/hyperlight_sandbox-0.9.0.tar.gz", hash = "sha256:248320cbdf116c4198d08228a0ada6b7697c3033894878a5c6cd577bce5d53d1", size = 13274, upload-time = "2026-11-06T23:32:21.933Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/22/522d2c6b186b66a5335e0fff99fe3ffdd82ce32b4968f9c9e951dd0a398b/hyperlight_sandbox-0.8.0-py3-none-any.whl", hash = "sha256:7aac916fa4dc1f630a72787d22082a1c1eed058ff866cb1d5ed53d2e9801f312", size = 6374, upload-time = "2026-10-17T17:19:56.869Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b0/f50c1afc6e544f57548bc3ab0ad6868021d2676707b33941c6a20a813bc9/hyperlight_sandbox-0.9.0-py3-none-any.whl", hash = "sha256:b953effd1f0d80385ceda74cd914ae841e03aefa5a30c051f7a85f4fb1c425ec", size = 6772, upload-time = "2026-11-06T23:32:21.050Z" }, ] [[package]] name = "hyperlight-sandbox-backend-wasm" -version = "0.8.0" +version = "0.9.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/8a/8f6f4dd003d0a84474844020c48ef7bc2996d77833d66b48b4959297bef6/hyperlight_sandbox_backend_wasm-0.8.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:412ebf5925e8caac3c367daa5bc5791c12a07260f1fb822d3d7e3806a74324ff", size = 3962518, upload-time = "2026-10-17T17:19:15.6Z" }, - { url = "https://files.pythonhosted.org/packages/91/80/640d247c206e38f24c5084e6d743ceb985d2e1a1e750f5806f29ba5bbe70/hyperlight_sandbox_backend_wasm-0.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:1c25db63ba5cdbcc8b68538f299c9634f59df4bfc5bedbdfcb321f0444a2c2dd", size = 3371142, upload-time = "2026-10-17T17:19:16.823Z" }, - { url = "https://files.pythonhosted.org/packages/7f/80/00af27dd716c60bfdbe999bf7ccf62e0b99fbcea0289350d98d0f1e747c5/hyperlight_sandbox_backend_wasm-0.8.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5ef64125c744ee81efe33a19065b71ee5466dce001db4e7789cd7d570bfc3f93", size = 3955761, upload-time = "2026-10-17T17:19:18.343Z" }, - { url = "https://files.pythonhosted.org/packages/5d/4b/acd51ca60f3461ec592959579705df86577e2c208233f34fcb493a77c67d/hyperlight_sandbox_backend_wasm-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:57b7c08091facc5451abec629a24864ca47d9d3ebafc5982052a1e29beae898a", size = 3367625, upload-time = "2026-10-17T17:19:19.948Z" }, - { url = "https://files.pythonhosted.org/packages/fe/30/9c5d4cbdb166d4cdd15d08f0de5dc5ef230d8cb055122eee9c6b2624d651/hyperlight_sandbox_backend_wasm-0.8.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6428ba7aa50d2a667d8579e1642a99da345fd8c3d7ae86a3dfb5dabad910bcf0", size = 3955707, upload-time = "2026-10-17T17:19:21.416Z" }, - { url = "https://files.pythonhosted.org/packages/0e/db/23c3e469b6d0a76a49f85a522aff08368abbffa87e47a2b0d76a18deb618/hyperlight_sandbox_backend_wasm-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:971605186775ba6004e50d91122c1c390e54f0e09e840e9e32587a2dff1d72ba", size = 3367557, upload-time = "2026-10-17T17:19:22.689Z" }, - { url = "https://files.pythonhosted.org/packages/aa/d9/176b9af30202bbf00bba12ca5df496dbee7ef402112c88b5c381359de5ef/hyperlight_sandbox_backend_wasm-0.8.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:bb71af32d64850f95a469f70922edcc151d524698d19e29e05af7a13075f711d", size = 3957485, upload-time = "2026-10-17T17:19:24.192Z" }, - { url = "https://files.pythonhosted.org/packages/90/77/92df8d875e4959092c6800027c1b058e73e515c76bbe1d1ab50bd09cc9e9/hyperlight_sandbox_backend_wasm-0.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:dfb927d2527057f6d21163c6e220a224e772a074cc5226a9bc6856dde5be462a", size = 3369173, upload-time = "2026-10-17T17:19:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/a6/94/38e0f2414f4076de55387c1d7ed0e097b38c846e91afdcb315a4cca5e537/hyperlight_sandbox_backend_wasm-0.9.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:80507cb8a11f80b1934c75cda940fd1f58afcd9a6f427a235082fd1c7caa4f28", size = 3953959, upload-time = "2026-11-06T23:31:26.811Z" }, + { url = "https://files.pythonhosted.org/packages/77/98/87cbcda610e239d61672c52aa6d1d02f86eaaf86b7ce7e45a3c8a1248e10/hyperlight_sandbox_backend_wasm-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:40493cfab0049726002262287889c0b6ef11ead29d18fb0a888866aff2d59e89", size = 3373392, upload-time = "2026-11-06T23:31:28.448Z" }, + { url = "https://files.pythonhosted.org/packages/cc/35/aea18ed7534c0de618ceb4c9cd0e013ecd192ccbf08d23bc9e1deea9978e/hyperlight_sandbox_backend_wasm-0.9.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:18f924eda5dc849f2bd7d707c325513d5ea8382fd812c4adc9b3cf81d39d56b9", size = 3946475, upload-time = "2026-11-06T23:31:29.872Z" }, + { url = "https://files.pythonhosted.org/packages/07/19/4116fd25f398b7f9c5e45fbfba20da2b74ce247bfa2308ceb80c8391cd3b/hyperlight_sandbox_backend_wasm-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:d29f49bc1613850cd54cf859fbef875fcd5050926e28b4167e526cc4084714a1", size = 3371812, upload-time = "2026-11-06T23:31:31.511Z" }, + { url = "https://files.pythonhosted.org/packages/e5/ca/547e777b3bfad3c0023a0069ad4000db6dddf5bd5a3f7d63dfdc09a05e40/hyperlight_sandbox_backend_wasm-0.9.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a565f420761b691e5868a76c4b05a6d90c9c3d709a7cac730c9317db0eead457", size = 3945855, upload-time = "2026-11-06T23:31:33.313Z" }, + { url = "https://files.pythonhosted.org/packages/46/d0/ecd8b9b1e01ddbcdd57623bf400b83da0e6f6b38abd6caebd253c728a764/hyperlight_sandbox_backend_wasm-0.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:2ef212d9df701890ff02489227bb085eadf42b502cfe279e8fcd2da81b27c79c", size = 3370762, upload-time = "2026-11-06T23:31:35.249Z" }, + { url = "https://files.pythonhosted.org/packages/f1/61/7418318bb5ceee70ff6df5315b4b562eaf9ad307b18b4d2b5d3cd7b21270/hyperlight_sandbox_backend_wasm-0.9.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:8515aa4caff2d656ca3ab46ef22512db5afeb8f5569797e03473710726be9fb5", size = 3945427, upload-time = "2026-11-06T23:31:36.825Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/a6011d16e37eae24e164d1b48098086a9d99c39f99bcfe72222b7f8f7162/hyperlight_sandbox_backend_wasm-0.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:57de522ba9593eefbf462b20f21b0a7f6d6ef61b4b66e196e3ffc6792c38752a", size = 3371484, upload-time = "2026-11-06T23:31:38.464Z" }, ] [[package]] name = "hyperlight-sandbox-python-guest" -version = "0.8.0" +version = "0.9.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/4d/3ffdfbe8c504f1cb1efd074c115117f3949316022152f4ef47065530348b/hyperlight_sandbox_python_guest-0.8.0.tar.gz", hash = "sha256:a0df229d2178cba3c04f7470958191ba17e00d2e18ce14500580dd8d179f7925", size = 21774726, upload-time = "2026-10-17T17:19:48.604Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/e2/eb7725da01cf89083f33107ad9f28e87ce94940c6663a1369539ef222b38/hyperlight_sandbox_python_guest-0.9.0.tar.gz", hash = "sha256:9636369028aa5a062ad7bcc8d2d32c7b2854b72b3bdabbe61e21676c4b2e4c02", size = 21767147, upload-time = "2026-11-06T23:32:09.934Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/89/65/a4629fceaa4b2a0f721ba26d7cc5a92123c046715dca3452d479a62680bb/hyperlight_sandbox_python_guest-0.8.0-py3-none-any.whl", hash = "sha256:fb2f27a2123e6368659540c3d845baf4b44d5493cbdb30bafa88287dbda46bbc", size = 21774618, upload-time = "2026-10-17T17:19:46.039Z" }, + { url = "https://files.pythonhosted.org/packages/5d/17/040c775b6c1c2d23a469b835a1d6ee0a269f16f7180a1a7ca2a7d3552acc/hyperlight_sandbox_python_guest-0.9.0-py3-none-any.whl", hash = "sha256:b04070e72c997e9b18e61639bb177d2b0945b3b963277a82bae0d4cb401d9eba", size = 21769679, upload-time = "2026-11-06T23:32:05.935Z" }, ] [[package]] From 9a487570e3f8eb88555a4a4e32ff679d73f50cf2 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Mon, 9 Nov 2026 13:00:45 -0500 Subject: [PATCH 3/6] chore: adds comment for exception pass through Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- .../hyperlight/agent_framework_hyperlight/_execute_code_tool.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py index 673f71a3692..71bbb3c7345 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py @@ -2039,6 +2039,7 @@ async def _run_code(self, *, code: str) -> list[Content]: try: execution.result() except asyncio.CancelledError: + # Expected during cancellation cleanup; the outer cancellation is re-raised below. pass except Exception: logger.warning( From 6e8ce9b01df05d39590eb50792dadc14b5d622ec Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Mon, 9 Nov 2026 13:28:11 -0500 Subject: [PATCH 4/6] Fix Hyperlight cold-start budget accounting and interruption coverage Exclude worker queue delays while retaining initialization costs, and cover timeout and cancellation cleanup during initialization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 69b80046-564c-406d-8a7a-85c3f7bd7f5d --- .../_execute_code_tool.py | 13 +- .../hyperlight/tests/hyperlight/conftest.py | 175 ++++++++++++++++++ .../hyperlight/test_execution_control.py | 74 ++++++++ 3 files changed, 260 insertions(+), 2 deletions(-) diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py index 71bbb3c7345..cdce28abaf8 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py @@ -91,16 +91,22 @@ class _GuestExecutionTimeout(TimeoutError): class _ExecutionControl: def __init__(self, timeout_seconds: float) -> None: - self._timeout_seconds = timeout_seconds + self._remaining_seconds = timeout_seconds self._deadline: float | None = None self._cancelled = _execution_cancellation.get() or threading.Event() def start(self) -> float: if self._deadline is None: - self._deadline = time.monotonic() + self._timeout_seconds + self._deadline = time.monotonic() + self._remaining_seconds self.check() return self._deadline + def pause(self) -> None: + self.check() + if self._deadline is not None: + self._remaining_seconds = self._deadline - time.monotonic() + self._deadline = None + def check(self) -> None: if self._cancelled.is_set(): raise asyncio.CancelledError @@ -1620,6 +1626,9 @@ def _build_sandbox() -> tuple[Any, Any]: control.run(sandbox, "None") snapshot = sandbox.snapshot() + # Initialization and execution are separate worker submissions; do not + # charge the invocation for queue time between them. + control.pause() return sandbox, snapshot worker = _SandboxWorker() diff --git a/python/packages/hyperlight/tests/hyperlight/conftest.py b/python/packages/hyperlight/tests/hyperlight/conftest.py index 21db573f11b..cdc608773cf 100644 --- a/python/packages/hyperlight/tests/hyperlight/conftest.py +++ b/python/packages/hyperlight/tests/hyperlight/conftest.py @@ -15,6 +15,181 @@ from agent_framework_hyperlight import _execute_code_tool as execute_code_module +@pytest.fixture +def execution_clock(monkeypatch: pytest.MonkeyPatch): + clock = SimpleNamespace(now=100.0) + monkeypatch.setattr(execute_code_module, "time", SimpleNamespace(monotonic=lambda: clock.now)) + return clock + + +@pytest.fixture +def interrupted_initialization(monkeypatch: pytest.MonkeyPatch, execution_clock, request: pytest.FixtureRequest): + stage = request.param + entered = threading.Event() + release = threading.Event() + interrupted = threading.Event() + cancelled = threading.Event() + events: list[tuple[int, str, int]] = [] + workers: list[Any] = [] + builds = 0 + + original_init = execute_code_module._SandboxWorker.__init__ + + def initialize_worker(self, **kwargs: Any) -> None: + original_init(self, **kwargs) + workers.append(self) + + class Snapshot: + def __init__(self, generation: int) -> None: + self.generation = generation + + def __del__(self) -> None: + events.append((self.generation, "drop_snapshot", threading.get_ident())) + + class Sandbox: + def __init__(self, **kwargs: Any) -> None: + nonlocal builds, cancelled + builds += 1 + self.generation = builds + events.append((self.generation, "build", threading.get_ident())) + if self.generation == 1: + cancellation = execute_code_module._execution_cancellation.get() + assert cancellation is not None + cancelled = cancellation + if self.generation == 1 and stage == "build": + entered.set() + if not release.wait(5): + raise RuntimeError("Sandbox construction was not released.") + + def interrupt_handle(self) -> SimpleNamespace: + def kill() -> bool: + interrupted.set() + return True + + return SimpleNamespace(kill=kill) + + def run(self, code: str) -> SimpleNamespace: + events.append((self.generation, "warmup" if code == "None" else "run", threading.get_ident())) + if self.generation == 1 and stage == "warmup": + entered.set() + if not interrupted.wait(5): + raise RuntimeError("Warm-up was not interrupted.") + raise InterruptedError("Warm-up interrupted.") + return SimpleNamespace(success=True, stdout="ok\n", stderr="") + + def snapshot(self) -> Snapshot: + snapshot = Snapshot(self.generation) + if self.generation == 1 and stage == "snapshot": + entered.set() + if not release.wait(5): + raise RuntimeError("Snapshot creation was not released.") + return snapshot + + def restore(self, snapshot: Snapshot) -> None: + events.append((self.generation, "restore", threading.get_ident())) + + def __del__(self) -> None: + events.append((self.generation, "drop", threading.get_ident())) + + monkeypatch.setattr(execute_code_module._SandboxWorker, "__init__", initialize_worker) + monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: Sandbox) + execute_code = HyperlightExecuteCodeTool(execution_timeout_seconds=10) + yield SimpleNamespace( + tool=execute_code, + clock=execution_clock, + entered=entered, + release=release, + interrupted=interrupted, + cancellation=lambda: cancelled, + events=events, + workers=workers, + stage=stage, + ) + release.set() + interrupted.set() + cast(Any, execute_code._registry).close() + + +@pytest.fixture +def concurrent_cold_execution(monkeypatch: pytest.MonkeyPatch, execution_clock, request: pytest.FixtureRequest): + initialized = threading.Event() + intervening_entered = threading.Event() + cold_queued = threading.Event() + interrupted = threading.Event() + executed: list[str] = [] + + class Sandbox: + def __init__(self, **kwargs: Any) -> None: + pass + + def interrupt_handle(self) -> SimpleNamespace: + def kill() -> bool: + interrupted.set() + return True + + return SimpleNamespace(kill=kill) + + def run(self, code: str) -> SimpleNamespace: + executed.append(code) + if code == "None": + execution_clock.now += 3 + elif code == "intervening": + intervening_entered.set() + if not cold_queued.wait(5): + raise RuntimeError("Cold invocation was not queued.") + execution_clock.now += 20 + else: + execution_clock.now += request.param + if request.param > 7: + if not interrupted.wait(5): + raise RuntimeError("Cold invocation was not interrupted.") + raise InterruptedError("Cold invocation interrupted.") + return SimpleNamespace(success=True, stdout="ok\n", stderr="") + + def snapshot(self) -> object: + return object() + + def restore(self, snapshot: object) -> None: + pass + + monkeypatch.setattr(execute_code_module, "_load_sandbox_class", lambda: Sandbox) + execute_code = HyperlightExecuteCodeTool(execution_timeout_seconds=10) + registry = cast(Any, execute_code._registry) + get_entry = registry._get_or_create_entry + + def get_entry_before_first_submission(config, *, control=None): + entry = get_entry(config, control=control) + if not initialized.is_set(): + cancellation = execute_code_module._execution_cancellation.get() + submit = entry.worker._executor.submit + + def submit_and_signal(fn, *args, **kwargs): + future = submit(fn, *args, **kwargs) + if execute_code_module._execution_cancellation.get() is cancellation: + cold_queued.set() + return future + + monkeypatch.setattr(entry.worker._executor, "submit", submit_and_signal) + initialized.set() + if not intervening_entered.wait(5): + raise RuntimeError("Intervening invocation did not start.") + return entry + + monkeypatch.setattr(registry, "_get_or_create_entry", get_entry_before_first_submission) + yield SimpleNamespace( + tool=execute_code, + initialized=initialized, + intervening_entered=intervening_entered, + cold_queued=cold_queued, + executed=executed, + duration=request.param, + ) + intervening_entered.set() + cold_queued.set() + interrupted.set() + registry.close() + + @pytest.fixture def interruptible_execution(monkeypatch: pytest.MonkeyPatch): entered = threading.Event() diff --git a/python/packages/hyperlight/tests/hyperlight/test_execution_control.py b/python/packages/hyperlight/tests/hyperlight/test_execution_control.py index 9118c97e732..0389fee3c98 100644 --- a/python/packages/hyperlight/tests/hyperlight/test_execution_control.py +++ b/python/packages/hyperlight/tests/hyperlight/test_execution_control.py @@ -3,6 +3,80 @@ from __future__ import annotations import asyncio +import threading + +import pytest + + +@pytest.mark.parametrize("concurrent_cold_execution", [6, 8], indirect=True) +async def test_cold_execution_excludes_queue_time_but_preserves_initialization_cost(concurrent_cold_execution) -> None: + native = concurrent_cold_execution + cold = asyncio.create_task(native.tool.invoke(arguments={"code": "cold"})) + try: + assert await asyncio.to_thread(native.initialized.wait, 2) + other_tool = native.tool.create_run_tool() + other_tool._execution_timeout_seconds = 100 + intervening = await asyncio.wait_for(other_tool.invoke(arguments={"code": "intervening"}), 2) + assert intervening[0].text == "ok\n" + assert native.cold_queued.is_set() + result = await asyncio.wait_for(cold, 2) + if native.duration > 7: + assert result[0].type == "error" + assert result[0].error_details == "Code execution exceeded its time limit." + else: + assert result[0].text == "ok\n" + assert native.executed == ["None", "intervening", "cold"] + finally: + native.intervening_entered.set() + native.cold_queued.set() + await asyncio.gather(cold, return_exceptions=True) + + +@pytest.mark.parametrize("interrupted_initialization", ["build", "warmup", "snapshot"], indirect=True) +@pytest.mark.parametrize("signal", ["timeout", "cancellation"]) +@pytest.mark.parametrize("include_detailed_errors", [False, True]) +async def test_initialization_interruption_cleans_up_and_allows_recreation( + interrupted_initialization, signal: str, include_detailed_errors: bool +) -> None: + native = interrupted_initialization + native.tool._include_detailed_errors = include_detailed_errors + execution = asyncio.create_task(native.tool.invoke(arguments={"code": "guest"})) + try: + assert await asyncio.to_thread(native.entered.wait, 2) + if signal == "timeout": + native.clock.now += 11 + else: + execution.cancel() + assert await asyncio.to_thread(native.cancellation().wait, 2) + native.release.set() + if signal == "timeout": + result = await asyncio.wait_for(execution, 2) + assert len(result) == 1 + assert result[0].type == "error" + assert result[0].error_details == "Code execution exceeded its time limit." + else: + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(execution, 2) + + assert native.tool._registry._entries == {} + assert not native.workers[0].is_alive() + first_events = [(stage, ident) for generation, stage, ident in native.events if generation == 1] + assert "drop" in [stage for stage, _ in first_events] + assert "run" not in [stage for stage, _ in first_events] + owner_threads = {ident for _, ident in first_events} + assert len(owner_threads) == 1 + assert threading.get_ident() not in owner_threads + if native.stage == "snapshot": + assert "drop_snapshot" in [stage for stage, _ in first_events] + + recovered = await asyncio.wait_for(native.tool.invoke(arguments={"code": "guest"}), 2) + assert recovered[0].text == "ok\n" + assert len(native.tool._registry._entries) == 1 + assert any(generation == 2 and stage == "run" for generation, stage, _ in native.events) + finally: + native.release.set() + native.interrupted.set() + await asyncio.gather(execution, return_exceptions=True) async def test_timeout_returns_error_and_allows_reuse(interruptible_execution) -> None: From 22954686ea43e9b459b4194408be8aad6c0aabbf Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Mon, 9 Nov 2026 14:39:44 -0500 Subject: [PATCH 5/6] Fix Hyperlight callback deadlines and queued cancellation Recheck execution controls after guest completion and cancel queued default-executor invocations without waiting for unrelated work. Add regressions for both cases and sandbox recovery. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 69b80046-564c-406d-8a7a-85c3f7bd7f5d --- .../_execute_code_tool.py | 22 ++++++-- .../hyperlight/tests/hyperlight/conftest.py | 45 ++++++++++++++-- .../hyperlight/test_execution_control.py | 54 +++++++++++++++++++ 3 files changed, 114 insertions(+), 7 deletions(-) diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py index cdce28abaf8..9db9cefbf4e 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py @@ -150,6 +150,7 @@ def _interrupt() -> None: interruption.join() if interruption_errors: raise RuntimeError("Could not interrupt Hyperlight guest execution.") + self.check() return result @@ -2029,13 +2030,26 @@ async def _run_code(self, *, code: str) -> list[Content]: current_context = contextvars.copy_context() finally: _execution_cancellation.reset(token) - execution = asyncio.get_running_loop().run_in_executor( - None, lambda: current_context.run(self._registry.execute, config=config, code=code) - ) + execution_started = False + start_lock = threading.Lock() + + def _execute() -> list[Content]: + nonlocal execution_started + with start_lock: + if cancelled.is_set(): + raise asyncio.CancelledError + execution_started = True + return current_context.run(self._registry.execute, config=config, code=code) + + execution = asyncio.get_running_loop().run_in_executor(None, _execute) try: return await asyncio.shield(execution) except asyncio.CancelledError: - cancelled.set() + with start_lock: + cancelled.set() + if not execution_started: + execution.cancel() + raise # Shield the owning worker until native execution and its interruption # thread have finished, including when the caller cancels again. while not execution.done(): diff --git a/python/packages/hyperlight/tests/hyperlight/conftest.py b/python/packages/hyperlight/tests/hyperlight/conftest.py index cdc608773cf..b167d95d15a 100644 --- a/python/packages/hyperlight/tests/hyperlight/conftest.py +++ b/python/packages/hyperlight/tests/hyperlight/conftest.py @@ -5,6 +5,7 @@ import asyncio import threading from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace from typing import Any, cast @@ -190,8 +191,44 @@ def submit_and_signal(fn, *args, **kwargs): registry.close() +@pytest.fixture +async def occupied_default_executor(monkeypatch: pytest.MonkeyPatch): + loop = asyncio.get_running_loop() + entered = asyncio.Event() + queued = asyncio.Event() + release = threading.Event() + executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="hl-test-default") + loop.set_default_executor(executor) + + def occupy_executor() -> None: + loop.call_soon_threadsafe(entered.set) + if not release.wait(5): + raise RuntimeError("Default executor was not released.") + + blocker = loop.run_in_executor(None, occupy_executor) + original_run_in_executor = loop.run_in_executor + + def run_in_executor(executor, func, *args): + future = original_run_in_executor(executor, func, *args) + if executor is None: + queued.set() + return future + + monkeypatch.setattr(loop, "run_in_executor", run_in_executor) + try: + await asyncio.wait_for(entered.wait(), 2) + yield SimpleNamespace(queued=queued, release=release, blocker=blocker) + finally: + release.set() + try: + await asyncio.wait_for(blocker, 2) + finally: + executor.shutdown(wait=True) + + @pytest.fixture def interruptible_execution(monkeypatch: pytest.MonkeyPatch): + started = threading.Event() entered = threading.Event() release = threading.Event() interrupted = threading.Event() @@ -200,6 +237,7 @@ def interruptible_execution(monkeypatch: pytest.MonkeyPatch): class Sandbox: def __init__(self, **kwargs: Any) -> None: + started.set() self._host_callback = False def interrupt_handle(self) -> SimpleNamespace: @@ -211,8 +249,8 @@ def kill() -> bool: return SimpleNamespace(kill=kill) def run(self, code: str) -> SimpleNamespace: - if code in {"guest", "host"}: - self._host_callback = code == "host" + if code in {"guest", "host", "final_host"}: + self._host_callback = code in {"host", "final_host"} entered.set() try: if self._host_callback: @@ -220,7 +258,7 @@ def run(self, code: str) -> SimpleNamespace: raise RuntimeError("Host callback was not released.") elif not interrupted.wait(5): raise RuntimeError("Guest was not interrupted.") - if interrupted.is_set(): + if interrupted.is_set() and code != "final_host": raise InterruptedError("Guest execution interrupted.") finally: completed.set() @@ -237,6 +275,7 @@ def restore(self, snapshot: object) -> None: execute_code = HyperlightExecuteCodeTool(execution_timeout_seconds=0.1) yield SimpleNamespace( tool=execute_code, + started=started, entered=entered, release=release, kill_attempted=kill_attempted, diff --git a/python/packages/hyperlight/tests/hyperlight/test_execution_control.py b/python/packages/hyperlight/tests/hyperlight/test_execution_control.py index 0389fee3c98..ce522bb9576 100644 --- a/python/packages/hyperlight/tests/hyperlight/test_execution_control.py +++ b/python/packages/hyperlight/tests/hyperlight/test_execution_control.py @@ -4,6 +4,7 @@ import asyncio import threading +from unittest.mock import Mock import pytest @@ -91,6 +92,59 @@ async def test_timeout_returns_error_and_allows_reuse(interruptible_execution) - assert recovered[0].text == "ok\n" +async def test_timeout_after_final_host_callback_returns_error_and_allows_reuse( + interruptible_execution, execution_clock +) -> None: + native = interruptible_execution + execution = asyncio.create_task(native.tool.invoke(arguments={"code": "final_host"})) + try: + assert await asyncio.to_thread(native.entered.wait, 2) + execution_clock.now += 1 + assert await asyncio.to_thread(native.kill_attempted.wait, 2) + assert not execution.done() + native.release.set() + result = await asyncio.wait_for(execution, 2) + assert native.completed.is_set() + assert len(result) == 1 + assert result[0].type == "error" + assert result[0].error_details == "Code execution exceeded its time limit." + recovered = await native.tool.invoke(arguments={"code": "None"}) + assert recovered[0].text == "ok\n" + finally: + native.release.set() + await asyncio.gather(execution, return_exceptions=True) + + +async def test_cancellation_before_default_executor_start_does_not_wait_or_create_sandbox( + interruptible_execution, occupied_default_executor, monkeypatch: pytest.MonkeyPatch +) -> None: + native = interruptible_execution + pool = occupied_default_executor + execute = Mock(wraps=native.tool._registry.execute) + monkeypatch.setattr(native.tool._registry, "execute", execute) + execution = asyncio.create_task(native.tool.invoke(arguments={"code": "None"})) + try: + await asyncio.wait_for(pool.queued.wait(), 2) + assert not native.started.is_set() + execution.cancel() + done, _ = await asyncio.wait({execution}, timeout=0.5) + assert done, "Queued cancellation waited for unrelated default-executor work." + with pytest.raises(asyncio.CancelledError): + await execution + assert not pool.blocker.done() + assert not native.started.is_set() + assert not native.kill_attempted.is_set() + finally: + pool.release.set() + await asyncio.gather(execution, return_exceptions=True) + await asyncio.wait_for(pool.blocker, 2) + await asyncio.get_running_loop().run_in_executor(None, lambda: None) + assert not native.started.is_set() + execute.assert_not_called() + recovered = await native.tool.invoke(arguments={"code": "None"}) + assert recovered[0].text == "ok\n" + + async def test_repeated_cancellation_waits_for_host_callback_to_finish(interruptible_execution) -> None: native = interruptible_execution execution = asyncio.create_task(native.tool.invoke(arguments={"code": "host"})) From e29c10c495190ba87d39f541aa0a51645d03fd9e Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Tue, 10 Nov 2026 11:45:53 -0500 Subject: [PATCH 6/6] Fix cancellation of queued Hyperlight worker invocations Cancel queued worker futures without waiting for another invocation's host callback, while retaining owner-thread cleanup once work starts. Verify cancellation finishes before the active callback is released and the sandbox remains reusable. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 69b80046-564c-406d-8a7a-85c3f7bd7f5d --- .../_execute_code_tool.py | 12 ++++++- .../hyperlight/tests/hyperlight/conftest.py | 21 ++++++++++++ .../hyperlight/test_execution_control.py | 32 ++++++++++++------- 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py index 9db9cefbf4e..94d946c2192 100644 --- a/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py +++ b/python/packages/hyperlight/agent_framework_hyperlight/_execute_code_tool.py @@ -307,7 +307,17 @@ def _wrapped() -> tuple[bool, Any]: return False, (exc_type, exc_args) current_context = contextvars.copy_context() - ok, payload = self._executor.submit(lambda: current_context.run(_wrapped)).result() + execution = self._executor.submit(lambda: current_context.run(_wrapped)) + cancelled = _execution_cancellation.get() + while True: + try: + ok, payload = execution.result(timeout=0.01 if cancelled is not None else None) + break + except TimeoutError: + # Future.cancel() atomically distinguishes queued work from work + # that has started and must finish cleanup on its owning thread. + if cancelled is not None and cancelled.is_set() and execution.cancel(): + raise asyncio.CancelledError from None if ok: return cast(_T, payload) exc_type, exc_args = cast(tuple[type[BaseException], tuple[str, ...]], payload) diff --git a/python/packages/hyperlight/tests/hyperlight/conftest.py b/python/packages/hyperlight/tests/hyperlight/conftest.py index b167d95d15a..e6a31398fdd 100644 --- a/python/packages/hyperlight/tests/hyperlight/conftest.py +++ b/python/packages/hyperlight/tests/hyperlight/conftest.py @@ -226,6 +226,24 @@ def run_in_executor(executor, func, *args): executor.shutdown(wait=True) +@pytest.fixture +def observe_sandbox_queue(monkeypatch: pytest.MonkeyPatch): + def observe(worker): + loop = asyncio.get_running_loop() + queued = asyncio.Event() + submit = worker._executor.submit + + def submit_and_signal(fn, *args, **kwargs): + future = submit(fn, *args, **kwargs) + loop.call_soon_threadsafe(queued.set) + return future + + monkeypatch.setattr(worker._executor, "submit", submit_and_signal) + return queued + + return observe + + @pytest.fixture def interruptible_execution(monkeypatch: pytest.MonkeyPatch): started = threading.Event() @@ -234,6 +252,7 @@ def interruptible_execution(monkeypatch: pytest.MonkeyPatch): interrupted = threading.Event() kill_attempted = threading.Event() completed = threading.Event() + executed: list[str] = [] class Sandbox: def __init__(self, **kwargs: Any) -> None: @@ -249,6 +268,7 @@ def kill() -> bool: return SimpleNamespace(kill=kill) def run(self, code: str) -> SimpleNamespace: + executed.append(code) if code in {"guest", "host", "final_host"}: self._host_callback = code in {"host", "final_host"} entered.set() @@ -280,6 +300,7 @@ def restore(self, snapshot: object) -> None: release=release, kill_attempted=kill_attempted, completed=completed, + executed=executed, ) release.set() cast(Any, execute_code._registry).close() diff --git a/python/packages/hyperlight/tests/hyperlight/test_execution_control.py b/python/packages/hyperlight/tests/hyperlight/test_execution_control.py index ce522bb9576..131cc673cbc 100644 --- a/python/packages/hyperlight/tests/hyperlight/test_execution_control.py +++ b/python/packages/hyperlight/tests/hyperlight/test_execution_control.py @@ -169,27 +169,35 @@ async def test_repeated_cancellation_waits_for_host_callback_to_finish(interrupt assert recovered[0].text == "ok\n" -async def test_queued_cancellation_does_not_interrupt_another_invocation(interruptible_execution) -> None: +async def test_queued_cancellation_does_not_wait_for_or_interrupt_another_invocation( + interruptible_execution, observe_sandbox_queue +) -> None: native = interruptible_execution - native.tool._execution_timeout_seconds = 5 + native.tool._execution_timeout_seconds = 60 active = asyncio.create_task(native.tool.invoke(arguments={"code": "host"})) queued = None try: assert await asyncio.to_thread(native.entered.wait, 2) - queued = asyncio.create_task(native.tool.invoke(arguments={"code": "None"})) - await asyncio.sleep(0.02) + entry = next(iter(native.tool._registry._entries.values())) + submitted = observe_sandbox_queue(entry.worker) + queued = asyncio.create_task(native.tool.invoke(arguments={"code": "guest"})) + await asyncio.wait_for(submitted.wait(), 2) queued.cancel() - await asyncio.sleep(0) + done, _ = await asyncio.wait({queued}, timeout=0.5) + assert done, "Queued cancellation waited for another invocation's host callback." + with pytest.raises(asyncio.CancelledError): + await queued + assert not active.done() + assert not native.release.is_set() + assert not native.completed.is_set() assert not native.kill_attempted.is_set() finally: native.release.set() result = await asyncio.wait_for(active, 2) + if queued is not None: + await asyncio.gather(queued, return_exceptions=True) assert result[0].text == "ok\n" - assert queued is not None - try: - await asyncio.wait_for(queued, 2) - except asyncio.CancelledError: - pass - else: - raise AssertionError("Queued cancellation was not preserved.") + recovered = await native.tool.invoke(arguments={"code": "None"}) + assert recovered[0].text == "ok\n" + assert native.executed == ["None", "host", "None"] assert not native.kill_attempted.is_set()