From 10ba905dc7bd567f0c26e5b659b626cdbfc7f36a Mon Sep 17 00:00:00 2001 From: Dillon DuPont Date: Sun, 11 May 2025 17:57:46 -0400 Subject: [PATCH] include selected text range --- .../computer_server/handlers/macos.py | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/libs/computer-server/computer_server/handlers/macos.py b/libs/computer-server/computer_server/handlers/macos.py index 10df3636..b76b28c7 100644 --- a/libs/computer-server/computer_server/handlers/macos.py +++ b/libs/computer-server/computer_server/handlers/macos.py @@ -35,6 +35,8 @@ from ApplicationServices import ( kAXRoleDescriptionAttribute, # type: ignore kAXFocusedApplicationAttribute, # type: ignore kAXFocusedUIElementAttribute, # type: ignore + kAXSelectedTextAttribute, # type: ignore + kAXSelectedTextRangeAttribute, # type: ignore ) import objc import re @@ -527,13 +529,8 @@ class MacOSAccessibilityHandler(BaseAccessibilityHandler): # Create system-wide accessibility object system = AXUIElementCreateSystemWide() - # Get focused application - err, focused_app = AXUIElementCopyAttributeValue(system, kAXFocusedApplicationAttribute, None) - if err != kAXErrorSuccess or not focused_app: - return {"success": False, "error": "Could not get focused application"} - - # Get focused UI element - err, focused_element = AXUIElementCopyAttributeValue(focused_app, kAXFocusedUIElementAttribute, None) + # Get focused element directly from system-wide element + err, focused_element = AXUIElementCopyAttributeValue(system, kAXFocusedUIElementAttribute, None) if err != kAXErrorSuccess or not focused_element: return {"success": False, "error": "Could not get focused UI element"} @@ -566,7 +563,16 @@ class MacOSAccessibilityHandler(BaseAccessibilityHandler): title = self.get_ax_attribute(focused_element, kAXTitleAttribute) value = self.get_ax_attribute(focused_element, kAXValueAttribute) - return { + # Check if there's selected text (as mentioned in the blog post) + selected_text = self.get_ax_attribute(focused_element, kAXSelectedTextAttribute) + + # Get selected text range if available + selected_range = None + selected_range_value = self.get_ax_attribute(focused_element, kAXSelectedTextRangeAttribute) + if selected_range_value: + selected_range = element_value(selected_range_value, kAXValueCFRangeType) + + result = { "success": True, "position": {"x": position_point.x, "y": position_point.y}, "size": {"width": size_value.width, "height": size_value.height}, @@ -576,6 +582,18 @@ class MacOSAccessibilityHandler(BaseAccessibilityHandler): "value": value } + # Add selected text information if available + if selected_text: + result["selected_text"] = selected_text + + if selected_range: + result["selected_range"] = { + "location": selected_range.location, + "length": selected_range.length + } + + return result + except Exception as e: return {"success": False, "error": str(e)}