Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ unfixable = []
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"

[tool.ruff.lint.per-file-ignores]
"src/askui/agent.py" = ["E501"]
"src/askui/computer_agent.py" = ["E501"]
"src/askui/android_agent.py" = ["E501"]
"src/askui/locators/locators.py" = ["E501"]
"src/askui/locators/relatable.py" = ["E501", "SLF001"]
Expand Down
2 changes: 1 addition & 1 deletion src/askui/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@

os.environ["FASTMCP_EXPERIMENTAL_ENABLE_NEW_OPENAPI_PARSER"] = "true"

from .agent import ComputerAgent, VisionAgent
from .agent_base import Agent
from .agent_settings import AgentSettings
from .computer_agent import ComputerAgent, VisionAgent
from .locators import Locator
from .models import (
Base64ImageSourceParam,
Expand Down
4 changes: 4 additions & 0 deletions src/askui/agent_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -878,3 +878,7 @@ def __exit__(
traceback: types.TracebackType | None,
) -> None:
self.close()

@staticmethod
def get_default_tools() -> list[Tool]:
return []
37 changes: 20 additions & 17 deletions src/askui/android_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,23 +81,7 @@ def __init__(
super().__init__(
reporter=reporter,
retry=retry,
tools=[
AndroidScreenshotTool(),
AndroidTapTool(),
AndroidTypeTool(),
AndroidDragAndDropTool(),
AndroidKeyTapEventTool(),
AndroidSwipeTool(),
AndroidKeyCombinationTool(),
AndroidShellTool(),
AndroidSelectDeviceBySerialNumberTool(),
AndroidSelectDisplayByUniqueIDTool(),
AndroidGetConnectedDevicesSerialNumbersTool(),
AndroidGetConnectedDisplaysInfosTool(),
AndroidGetCurrentConnectedDeviceInfosTool(),
ExceptionTool(),
]
+ (act_tools or []),
tools=self.get_default_tools() + (act_tools or []),
agent_os=self.os,
settings=settings,
callbacks=callbacks,
Expand Down Expand Up @@ -357,6 +341,25 @@ def set_device_by_serial_number(
)
self.os.set_device_by_serial_number(device_sn)

@staticmethod
def get_default_tools() -> list[Tool]:
return [
AndroidScreenshotTool(),
AndroidTapTool(),
AndroidTypeTool(),
AndroidDragAndDropTool(),
AndroidKeyTapEventTool(),
AndroidSwipeTool(),
AndroidKeyCombinationTool(),
AndroidShellTool(),
AndroidSelectDeviceBySerialNumberTool(),
AndroidSelectDisplayByUniqueIDTool(),
AndroidGetConnectedDevicesSerialNumbersTool(),
AndroidGetConnectedDisplaysInfosTool(),
AndroidGetCurrentConnectedDeviceInfosTool(),
ExceptionTool(),
]


class AndroidVisionAgent(AndroidAgent):
def __init__(self, *args, **kwargs) -> None: # type: ignore
Expand Down
41 changes: 22 additions & 19 deletions src/askui/agent.py → src/askui/computer_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,25 +92,7 @@ def __init__(
super().__init__(
reporter=reporter,
retry=retry,
tools=[
ExceptionTool(),
ComputerGetSystemInfoTool(),
ComputerGetMousePositionTool(),
ComputerKeyboardPressedTool(),
ComputerKeyboardReleaseTool(),
ComputerKeyboardTapTool(),
ComputerMouseClickTool(),
ComputerMouseHoldDownTool(),
ComputerMouseReleaseTool(),
ComputerMouseScrollTool(),
ComputerMoveMouseTool(),
ComputerScreenshotTool(),
ComputerTypeTool(),
ComputerListDisplaysTool(),
ComputerRetrieveActiveDisplayTool(),
ComputerSetActiveDisplayTool(),
]
+ (act_tools or []),
tools=self.get_default_tools() + (act_tools or []),
agent_os=self.tools.os,
settings=settings,
callbacks=callbacks,
Expand Down Expand Up @@ -508,6 +490,27 @@ def cli(
)
self.tools.os.run_command(command)

@staticmethod
def get_default_tools() -> list[Tool]:
return [
ExceptionTool(),
ComputerGetSystemInfoTool(),
ComputerGetMousePositionTool(),
ComputerKeyboardPressedTool(),
ComputerKeyboardReleaseTool(),
ComputerKeyboardTapTool(),
ComputerMouseClickTool(),
ComputerMouseHoldDownTool(),
ComputerMouseReleaseTool(),
ComputerMouseScrollTool(),
ComputerMoveMouseTool(),
ComputerScreenshotTool(),
ComputerTypeTool(),
ComputerListDisplaysTool(),
ComputerRetrieveActiveDisplayTool(),
ComputerSetActiveDisplayTool(),
]


class VisionAgent(ComputerAgent):
def __init__(self, *args, **kwargs) -> None: # type: ignore
Expand Down
2 changes: 1 addition & 1 deletion src/askui/web_agent.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from pydantic import ConfigDict, validate_call

from askui.agent import ComputerAgent
from askui import ComputerAgent
from askui.agent_settings import AgentSettings
from askui.models.shared.settings import (
ActSettings,
Expand Down
3 changes: 1 addition & 2 deletions tests/e2e/agent/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@
import pytest
from typing_extensions import override

from askui import AgentSettings
from askui.agent import ComputerAgent
from askui import AgentSettings, ComputerAgent
from askui.locators.locators import Locator
from askui.locators.serializers import AskUiLocatorSerializer
from askui.model_providers.detection_provider import DetectionProvider
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/agent/test_act.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from askui.agent import ComputerAgent
from askui import ComputerAgent


def test_act(
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/agent/test_act_caching.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import tempfile
from pathlib import Path

from askui.agent import ComputerAgent
from askui import ComputerAgent
from askui.models.shared.settings import CacheExecutionSettings, CachingSettings


Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/agent/test_agent_offset.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from PIL import Image as PILImage

from askui.agent import ComputerAgent
from askui import ComputerAgent
from askui.locators import Element

if TYPE_CHECKING:
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/agent/test_locate.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import pytest
from PIL import Image as PILImage

from askui.agent import ComputerAgent
from askui import ComputerAgent
from askui.locators import AiElement, Element, Prompt, Text
from askui.locators.locators import Image
from askui.models.exceptions import ElementNotFoundError
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/agent/test_locate_with_different_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import pytest
from PIL import Image as PILImage

from askui.agent import ComputerAgent
from askui import ComputerAgent
from askui.exceptions import AutomationError
from askui.locators import AiElement, Prompt, Text

Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/agent/test_locate_with_relations.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import pytest
from PIL import Image as PILImage

from askui.agent import ComputerAgent
from askui import ComputerAgent
from askui.locators import Element, Image, Prompt, Text
from askui.locators.locators import AiElement
from askui.models.exceptions import ElementNotFoundError
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/agent/test_wait.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from PIL import Image as PILImage
from pydantic import ValidationError

from askui.agent import ComputerAgent
from askui import ComputerAgent
from askui.locators import Element
from askui.models.exceptions import WaitUntilError

Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/test_telemetry.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import pytest
from PIL import Image

from askui import ComputerAgent
from askui import locators as loc
from askui.agent import ComputerAgent
from askui.container import telemetry
from askui.telemetry.processors import Segment, SegmentSettings
from askui.tools.toolbox import AgentToolbox
Expand Down