Skip to content

feat(core): maref 核心库/sidecar/loop 增强 #369

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
frankiehot-tech wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat(core): maref 核心库/sidecar/loop 增强 #369

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 30 additions & 30 deletions src/maref/desktop/accessibility_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,11 +128,14 @@
}
return;
}
var title = String(el.title());
var desc = String(el.description());
var value = String(el.value());
var pos = el.position();
var size = el.size();
// 单属性容错：某些 AXUIElement 不支持 title/position 等，抛错不应丢整棵子树。
var title = ""; var desc = ""; var value = "";
try { title = String(el.title()); } catch(e) {}
try { desc = String(el.description()); } catch(e) {}
try { value = String(el.value()); } catch(e) {}
var pos = [0, 0]; var size = [0, 0];
try { pos = el.position(); } catch(e) {}
try { size = el.size(); } catch(e) {}
var enabled = true;
try { enabled = el.enabled(); } catch(e) {}
var focused = false;
Expand Down Expand Up @@ -210,7 +213,6 @@ def check_permissions(self) -> bool:
return False
try:
import ApplicationServices

trusted = ApplicationServices.AXIsProcessTrusted()
self._permission_granted = bool(trusted)
return self._permission_granted
Expand Down Expand Up @@ -244,10 +246,11 @@ def parse(
ScreenParseResult with real UI elements from the Accessibility tree.
"""
if not self._initialized:
raise RuntimeError("AccessibilityParser not initialized. Call initialize() first.")
raise RuntimeError(
"AccessibilityParser not initialized. Call initialize() first."
)

import time

t0 = time.time()

raw_json = self._run_jxa(target_app)
Expand All @@ -261,7 +264,6 @@ def parse(
if width <= 0 or height <= 0:
try:
import Quartz

main_display = Quartz.CGMainDisplayID()
width = int(Quartz.CGDisplayPixelsWide(main_display))
height = int(Quartz.CGDisplayPixelsHigh(main_display))
Expand Down Expand Up @@ -295,22 +297,20 @@ def parse(
continue
interactions = _ROLE_INTERACTIONS.get(role, [])
elem_id = f"ax_{role}_{x}_{y}_{w}_{h}"
elements.append(
ParsedUIElement(
element_type=element_type,
bbox=BoundingBox(x=x, y=y, width=w, height=h),
text=title,
confidence=1.0,
interaction_types=interactions,
element_id=elem_id,
attributes={
"ax_role": role,
"enabled": is_enabled,
"focused": item.get("focused", False),
"source": "accessibility_api",
},
)
)
elements.append(ParsedUIElement(
element_type=element_type,
bbox=BoundingBox(x=x, y=y, width=w, height=h),
text=title,
confidence=1.0,
interaction_types=interactions,
element_id=elem_id,
attributes={
"ax_role": role,
"enabled": is_enabled,
"focused": item.get("focused", False),
"source": "accessibility_api",
},
))

elapsed = (time.time() - t0) * 1000
return ScreenParseResult(
Expand All @@ -324,14 +324,12 @@ def parse(

def _run_jxa(self, target_app: str) -> str:
"""Execute JXA script and return JSON element array."""
sanitized_app = re.sub(r"[^a-zA-Z0-9\- ]", "", target_app)
sanitized_app = re.sub(r'[^a-zA-Z0-9\- ]', '', target_app)
script = _JXA_GET_ELEMENTS % (sanitized_app, self._max_depth)
try:
result = subprocess.run(
["osascript", "-l", "JavaScript", "-e", script],
capture_output=True,
text=True,
timeout=15,
capture_output=True, text=True, timeout=15,
)
if result.returncode == 0:
return result.stdout.strip()
Expand All @@ -343,7 +341,9 @@ def _run_jxa(self, target_app: str) -> str:
def benchmark(self, num_runs: int = 3) -> dict[str, Any]:
"""Run a quick benchmark: avg latency and element count."""
if not self._initialized:
raise RuntimeError("AccessibilityParser not initialized. Call initialize() first.")
raise RuntimeError(
"AccessibilityParser not initialized. Call initialize() first."
)
latencies: list[float] = []
counts: list[int] = []
for _ in range(num_runs):
Expand Down
Loading