diff --git a/libs/python/agent/agent/human_tool/ui.py b/libs/python/agent/agent/human_tool/ui.py index d9d9ab11..34e1c468 100644 --- a/libs/python/agent/agent/human_tool/ui.py +++ b/libs/python/agent/agent/human_tool/ui.py @@ -196,7 +196,9 @@ class HumanCompletionUI: gr.update(choices=["latest"], value="latest"), # dropdown gr.update(value=None), # image (no image) gr.update(value=[]), # chatbot (empty messages) - gr.update(interactive=False) # submit button + gr.update(interactive=False), # submit button + gr.update(visible=False), # click_actions_group hidden + gr.update(visible=False), # actions_group hidden ) # Sort pending calls by created_at to get oldest first @@ -237,7 +239,9 @@ class HumanCompletionUI: gr.update(choices=choices, value="latest"), gr.update(value=self.last_image), gr.update(value=conversation), - gr.update(interactive=bool(choices)) + gr.update(interactive=bool(choices)), + gr.update(visible=True), # click_actions_group visible when there is a call + gr.update(visible=True), # actions_group visible when there is a call ) def on_call_selected(self, selected_choice): @@ -246,7 +250,9 @@ class HumanCompletionUI: return ( gr.update(value=None), # no image gr.update(value=[]), # empty chatbot - gr.update(interactive=False) + gr.update(interactive=False), + gr.update(visible=False), # click_actions_group hidden + gr.update(visible=False), # actions_group hidden ) pending_calls = self.get_pending_calls() @@ -254,7 +260,9 @@ class HumanCompletionUI: return ( gr.update(value=None), # no image gr.update(value=[]), # empty chatbot - gr.update(interactive=False) + gr.update(interactive=False), + gr.update(visible=False), # click_actions_group hidden + gr.update(visible=False), # actions_group hidden ) # Handle "latest" option @@ -286,7 +294,9 @@ class HumanCompletionUI: return ( gr.update(value=None), # no image gr.update(value=[]), # empty chatbot - gr.update(interactive=False) + gr.update(interactive=False), + gr.update(visible=False), # click_actions_group hidden + gr.update(visible=False), # actions_group hidden ) conversation = self.format_messages_for_chatbot(selected_call.get("messages", [])) @@ -297,7 +307,9 @@ class HumanCompletionUI: return ( gr.update(value=self.last_image), gr.update(value=conversation), - gr.update(interactive=True) + gr.update(interactive=True), + gr.update(visible=True), # click_actions_group visible + gr.update(visible=True), # actions_group visible ) def submit_response(self, response_text: str): @@ -411,7 +423,7 @@ def create_ui(): """Create the Gradio interface.""" ui_handler = HumanCompletionUI() - with gr.Blocks(title="Human-in-the-Loop Agent Tool") as demo: + with gr.Blocks(title="Human-in-the-Loop Agent Tool", fill_width=True) as demo: gr.Markdown("# 🤖 Human-in-the-Loop Agent Tool") gr.Markdown("Review AI conversation requests and provide human responses.") @@ -424,21 +436,22 @@ def create_ui(): height=600 ) - # Action type selection for image clicks - with gr.Row(): - action_type_radio = gr.Radio( - label="Action Type", - choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"], - value="click", - scale=2 - ) - action_button_radio = gr.Radio( - label="Button (for click only)", - choices=["left", "right", "wheel", "back", "forward"], - value="left", - visible=True, - scale=1 - ) + # Action type selection for image clicks (wrapped for visibility control) + with gr.Group(visible=False) as click_actions_group: + with gr.Row(): + action_type_radio = gr.Dropdown( + label="Interactive Action", + choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"], + value="click", + scale=2 + ) + action_button_radio = gr.Dropdown( + label="Button", + choices=["left", "right", "wheel", "back", "forward"], + value="left", + visible=True, + scale=1 + ) conversation_chatbot = gr.Chatbot( label="Messages", @@ -450,12 +463,17 @@ def create_ui(): with gr.Column(scale=1): with gr.Group(): call_dropdown = gr.Dropdown( - label="Select a pending call", + label="Select a pending conversation request", choices=["latest"], interactive=True, value="latest" ) refresh_btn = gr.Button("🔄 Refresh", variant="secondary") + status_display = gr.Textbox( + label="Status", + interactive=False, + value="Ready to receive requests..." + ) with gr.Group(): response_text = gr.Textbox( @@ -464,68 +482,64 @@ def create_ui(): placeholder="Enter your message here..." ) submit_btn = gr.Button("📤 Submit Message", variant="primary", interactive=False) - status_display = gr.Textbox( - label="Status", - interactive=False, - value="Ready to receive calls..." - ) - # Action Accordions - with gr.Tabs(): - with gr.Tab("🖱️ Click Actions"): - with gr.Group(): - description_text = gr.Textbox( - label="Element Description", - placeholder="e.g., 'Privacy and security option in left sidebar'" - ) - with gr.Row(): - description_action_type = gr.Dropdown( - label="Action Type", - choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"], - value="click" + # Action Accordions (wrapped for visibility control) + with gr.Group(visible=False) as actions_group: + with gr.Tabs(): + with gr.Tab("🖱️ Click Actions"): + with gr.Group(): + description_text = gr.Textbox( + label="Element Description", + placeholder="e.g., 'Privacy and security option in left sidebar'" ) - description_button = gr.Radio( - label="Button (for click only)", - choices=["left", "right", "wheel", "back", "forward"], - value="left" + with gr.Row(): + description_action_type = gr.Dropdown( + label="Action", + choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"], + value="click" + ) + description_button = gr.Dropdown( + label="Button", + choices=["left", "right", "wheel", "back", "forward"], + value="left" + ) + description_submit_btn = gr.Button("Submit Click Action") + + with gr.Tab("📝 Type Action"): + with gr.Group(): + type_text = gr.Textbox( + label="Text to Type", + placeholder="Enter text to type..." ) - description_submit_btn = gr.Button("Submit Click Action") - - with gr.Tab("📝 Type Action"): - with gr.Group(): - type_text = gr.Textbox( - label="Text to Type", - placeholder="Enter text to type..." - ) - type_submit_btn = gr.Button("Submit Type") - - with gr.Tab("⌨️ Keypress Action"): - with gr.Group(): - keypress_text = gr.Textbox( - label="Keys", - placeholder="e.g., ctrl+c, alt+tab" - ) - keypress_submit_btn = gr.Button("Submit Keypress") - - with gr.Tab("🧰 Misc Actions"): - with gr.Group(): - misc_action_dropdown = gr.Dropdown( - label="Action", - choices=["wait"], - value="wait" - ) - misc_submit_btn = gr.Button("Submit Action") + type_submit_btn = gr.Button("Submit Type") + + with gr.Tab("⌨️ Keypress Action"): + with gr.Group(): + keypress_text = gr.Textbox( + label="Keys", + placeholder="e.g., ctrl+c, alt+tab" + ) + keypress_submit_btn = gr.Button("Submit Keypress") + + with gr.Tab("🧰 Misc Actions"): + with gr.Group(): + misc_action_dropdown = gr.Dropdown( + label="Action", + choices=["wait"], + value="wait" + ) + misc_submit_btn = gr.Button("Submit Action") # Event handlers refresh_btn.click( fn=ui_handler.refresh_pending_calls, - outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn] + outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group] ) call_dropdown.change( fn=ui_handler.on_call_selected, inputs=[call_dropdown], - outputs=[screenshot_image, conversation_chatbot, submit_btn] + outputs=[screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group] ) def handle_image_click(evt: gr.SelectData): @@ -543,7 +557,7 @@ def create_ui(): outputs=[status_display] ).then( fn=ui_handler.wait_for_pending_calls, - outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn] + outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group] ) # Response submission @@ -553,7 +567,7 @@ def create_ui(): outputs=[response_text, status_display] ).then( fn=ui_handler.refresh_pending_calls, - outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn] + outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group] ) # Toggle button radio visibility based on action type @@ -572,7 +586,7 @@ def create_ui(): outputs=[status_display] ).then( fn=ui_handler.wait_for_pending_calls, - outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn] + outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group] ) keypress_submit_btn.click( @@ -581,7 +595,7 @@ def create_ui(): outputs=[status_display] ).then( fn=ui_handler.wait_for_pending_calls, - outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn] + outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group] ) def handle_description_submit(description, action_type, button): @@ -597,7 +611,7 @@ def create_ui(): outputs=[status_display] ).then( fn=ui_handler.wait_for_pending_calls, - outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn] + outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group] ) # Misc action handler @@ -614,13 +628,13 @@ def create_ui(): outputs=[status_display] ).then( fn=ui_handler.wait_for_pending_calls, - outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn] + outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group] ) # Load initial data demo.load( fn=ui_handler.refresh_pending_calls, - outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn] + outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group] ) return demo