Merge branch 'main' into feat/extra-models

libs/python/agent/agent/adapters/huggingfacelocal_adapter.py

@@ -8,7 +8,7 @@ from litellm import completion, acompletion
 # Try to import HuggingFace dependencies
 try:
     import torch
-    from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor
+    from transformers import AutoModelForImageTextToText, AutoProcessor
     HF_AVAILABLE = True
 except ImportError:
     HF_AVAILABLE = False
@@ -40,7 +40,7 @@ class HuggingFaceLocalAdapter(CustomLLM):
         """
         if model_name not in self.models:
             # Load model
-            model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+            model = AutoModelForImageTextToText.from_pretrained(
                 model_name,
                 torch_dtype=torch.float16,
                 device_map=self.device,
@@ -145,8 +145,7 @@ class HuggingFaceLocalAdapter(CustomLLM):
         )
         
         # Move inputs to the same device as model
-        if torch.cuda.is_available() and self.device != "cpu":
-            inputs = inputs.to("cuda")
+        inputs = inputs.to(model.device)
         
         # Generate response
         with torch.no_grad():

libs/python/agent/agent/agent.py

@@ -422,6 +422,9 @@ class ComputerAgent:
             # Perform computer actions
             action = item.get("action")
             action_type = action.get("type")
+            if action_type is None:
+                print(f"Action type cannot be `None`: action={action}, action_type={action_type}")
+                return []
             
             # Extract action arguments (all fields except 'type')
             action_args = {k: v for k, v in action.items() if k != "type"}

libs/python/agent/agent/callbacks/pii_anonymization.py

@@ -93,4 +93,4 @@ class PIIAnonymizationCallback(AsyncCallbackHandler):
     
     async def _deanonymize_item(self, item: Dict[str, Any]) -> Dict[str, Any]:
         # TODO: Implement _deanonymize_item
-        return item
+        return item

libs/python/agent/agent/ui/gradio/app.py

@@ -178,13 +178,20 @@ def create_computer_instance(
     """Create or get the global Computer instance."""
     global global_computer
     if global_computer is None:
-        global_computer = Computer(
-            verbosity=verbosity,
-            os_type=os_type,
-            provider_type=provider_type,
-            name=name if name else "",
-            api_key=api_key
-        )
+        if provider_type == "localhost":
+            global_computer = Computer(
+                verbosity=verbosity,
+                os_type=os_type,
+                use_host_computer_server=True
+            )
+        else:
+            global_computer = Computer(
+                verbosity=verbosity,
+                os_type=os_type,
+                provider_type=provider_type,
+                name=name if name else "",
+                api_key=api_key
+            )
     return global_computer
 
 

libs/python/agent/agent/ui/gradio/ui_components.py

@@ -211,7 +211,7 @@ if __name__ == "__main__":
                     is_windows = platform.system().lower() == "windows"
                     is_mac = platform.system().lower() == "darwin"
                     
-                    providers = ["cloud"]
+                    providers = ["cloud", "localhost"]
                     if is_mac:
                         providers += ["lume"]
                     if is_windows:
@@ -403,6 +403,23 @@ if __name__ == "__main__":
                         type="password",
                     )
                     
+                    # Provider visibility update function
+                    def update_provider_visibility(provider):
+                        """Update visibility of container name and API key based on selected provider."""
+                        is_localhost = provider == "localhost"
+                        return [
+                            gr.update(visible=not is_localhost),  # container_name
+                            gr.update(visible=not is_localhost and not has_cua_key)  # cua_cloud_api_key
+                        ]
+                    
+                    # Connect provider change event
+                    computer_provider.change(
+                        fn=update_provider_visibility,
+                        inputs=[computer_provider],
+                        outputs=[container_name, cua_cloud_api_key],
+                        queue=False
+                    )
+                    
                     # Connect UI update events
                     for dropdown in [agent_loop, omni_model_choice, uitars_model_choice, openai_model_choice, anthropic_model_choice]:
                         dropdown.change(

libs/python/agent/pyproject.toml

@@ -19,7 +19,7 @@ dependencies = [
     "pydantic>=2.6.4",
     "rich>=13.7.1",
     "python-dotenv>=1.0.1",
-    "cua-computer>=0.3.0,<0.5.0",
+    "cua-computer>=0.4.0,<0.5.0",
     "cua-core>=0.1.8,<0.2.0",
     "certifi>=2024.2.2",
     "litellm>=1.74.12"