diff --git a/libs/computer-server/computer_server/handlers/macos.py b/libs/computer-server/computer_server/handlers/macos.py index a4ad6f5f..8664179b 100644 --- a/libs/computer-server/computer_server/handlers/macos.py +++ b/libs/computer-server/computer_server/handlers/macos.py @@ -134,6 +134,9 @@ class UIElement: self.role_description = "" self.value = None self.max_depth = max_depth + self.selected_text = None + self.selected_range = None + self.focused = False # Set role self.role = element_attribute(element, kAXRoleAttribute) @@ -152,6 +155,21 @@ class UIElement: self.enabled = element_attribute(element, kAXEnabledAttribute) if self.enabled is None: self.enabled = False + + # Check if element is focused - using a simpler approach + try: + # Get the focused state directly from the element if available + # focused_attr = element_attribute(element, "AXFocused") + # if focused_attr is not None and focused_attr is True: + # self.focused = True + + # Get the focused element + focused_element = element_attribute(element, kAXFocusedUIElementAttribute) + if focused_element and Foundation.CFEqual(focused_element, element): + self.focused = True + except: + # If there's an error, assume it's not focused + pass # Set position and size position = element_attribute(element, kAXPositionAttribute) @@ -194,6 +212,14 @@ class UIElement: # Check if it's an accessibility element by checking its type ID elif Foundation.CFGetTypeID(attribute_value) == AXUIElementGetTypeID(): # type: ignore self.value = UIElement(attribute_value, offset_x, offset_y) + + # Get selected text if available + self.selected_text = element_attribute(element, kAXSelectedTextAttribute) + + # Get selected text range if available + selected_range_value = element_attribute(element, kAXSelectedTextRangeAttribute) + if selected_range_value: + self.selected_range = element_value(selected_range_value, kAXValueCFRangeType) # Set children if self.max_depth is None or self.max_depth > 0: @@ -321,8 +347,8 @@ class UIElement: size = f"{self.size.width:.0f};{self.size.height:.0f}" else: size = "" - - return { + + result = { "id": self.identifier, "name": self.name, "role": self.role, @@ -333,10 +359,23 @@ class UIElement: "position": position, "size": size, "enabled": self.enabled, + "focused": self.focused, "bbox": self.bbox, "visible_bbox": self.visible_bbox, "children": children_to_dict(self.children), } + + # Add selected text information if available + if self.selected_text: + result["selected_text"] = self.selected_text + + if self.selected_range: + result["selected_range"] = { + "location": self.selected_range.location, + "length": self.selected_range.length + } + + return result class MacOSAccessibilityHandler(BaseAccessibilityHandler):