diff --git a/examples/agent_examples.py b/examples/agent_examples.py index 6d7aa796..2baa0b70 100644 --- a/examples/agent_examples.py +++ b/examples/agent_examples.py @@ -32,9 +32,9 @@ async def run_agent_example(): # loop=AgentLoop.ANTHROPIC, loop=AgentLoop.OMNI, # model=LLM(provider=LLMProvider.OPENAI), # No model name for Operator CUA - # model=LLM(provider=LLMProvider.OPENAI, name="gpt-4.5-preview"), - # model=LLM(provider=LLMProvider.ANTHROPIC, name="claude-3-7-sonnet-20250219"), - model=LLM(provider=LLMProvider.OLLAMA, name="gemma3:4b-it-q4_K_M"), + # model=LLM(provider=LLMProvider.OPENAI, name="gpt-4o"), + model=LLM(provider=LLMProvider.ANTHROPIC, name="claude-3-7-sonnet-20250219"), + # model=LLM(provider=LLMProvider.OLLAMA, name="gemma3:4b-it-q4_K_M"), # model=LLM( # provider=LLMProvider.OAICOMPAT, # name="qwen2.5-vl-7b-instruct", diff --git a/libs/agent/README.md b/libs/agent/README.md index 7bbd3656..1d0d5df5 100644 --- a/libs/agent/README.md +++ b/libs/agent/README.md @@ -109,6 +109,33 @@ export ANTHROPIC_API_KEY=your_anthropic_key_here OPENAI_API_KEY=your_key ANTHROPIC_API_KEY=your_key python launch_ui.py ``` +### Using Local Models + +You can use local models with the OMNI loop provider by selecting "Custom model..." from the dropdown. The default provider URL is set to `http://localhost:1234/v1` which works with LM Studio. + +If you're using a different local model server: +- vLLM: `http://localhost:8000/v1` +- LocalAI: `http://localhost:8080/v1` +- Ollama with OpenAI compat API: `http://localhost:11434/v1` + +To change the URL, modify the `provider_base_url` in your launcher script: + +```python +# In your launcher script +from agent.ui.gradio.app import create_gradio_ui +from agent import LLM, LLMProvider + +# Create a custom model with a specific URL +custom_model = LLM( + provider=LLMProvider.OAICOMPAT, + name="your-model-name", + provider_base_url="http://localhost:8000/v1" # Change to your server URL +) + +app = create_gradio_ui(custom_model=custom_model) +app.launch() +``` + Without these environment variables, the UI will show "No models available" for the corresponding providers, but you can still use local models with the OMNI loop provider. The Gradio UI provides: @@ -123,14 +150,8 @@ You can also embed the Gradio UI in your own application: # Import directly in your application from agent.ui.gradio.app import create_gradio_ui -# Create the UI with advanced features -demo = create_gradio_ui() -demo.launch() - -# Or for a simpler interface -from agent.ui.gradio import registry -demo = registry(name='cua:gpt-4o') -demo.launch() +app = create_gradio_ui() +app.launch() ``` ## Agent Loops @@ -141,7 +162,7 @@ The `cua-agent` package provides three agent loops variations, based on differen |:-----------|:-----------------|:------------|:-------------| | `AgentLoop.OPENAI` | • `computer_use_preview` | Use OpenAI Operator CUA model | Not Required | | `AgentLoop.ANTHROPIC` | • `claude-3-5-sonnet-20240620`
• `claude-3-7-sonnet-20250219` | Use Anthropic Computer-Use | Not Required | -| `AgentLoop.OMNI` | • `claude-3-5-sonnet-20240620`
• `claude-3-7-sonnet-20250219`
• `gpt-4.5-preview`
• `gpt-4o`
• `gpt-4`
• `phi4`
• `phi4-mini`
• `gemma3`
• `...`
• `Any Ollama-compatible model` | Use OmniParser for element pixel-detection (SoM) and any VLMs for UI Grounding and Reasoning | OmniParser | +| `AgentLoop.OMNI` | • `claude-3-5-sonnet-20240620`
• `claude-3-7-sonnet-20250219`
• `gpt-4.5-preview`
• `gpt-4o`
• `gpt-4`
• `phi4`
• `phi4-mini`
• `gemma3`
• `...`
• `Any Ollama or OpenAI-compatible model` | Use OmniParser for element pixel-detection (SoM) and any VLMs for UI Grounding and Reasoning | OmniParser | ## AgentResponse The `AgentResponse` class represents the structured output returned after each agent turn. It contains the agent's response, reasoning, tool usage, and other metadata. The response format aligns with the new [OpenAI Agent SDK specification](https://platform.openai.com/docs/api-reference/responses) for better consistency across different agent loops. diff --git a/libs/agent/agent/providers/omni/loop.py b/libs/agent/agent/providers/omni/loop.py index 33124b25..7fb80654 100644 --- a/libs/agent/agent/providers/omni/loop.py +++ b/libs/agent/agent/providers/omni/loop.py @@ -443,6 +443,8 @@ class OmniLoop(BaseLoop): except (json.JSONDecodeError, IndexError): try: # Look for JSON object pattern + import re # Local import to ensure availability + json_pattern = r"\{[^}]+\}" json_match = re.search(json_pattern, raw_text) if json_match: @@ -453,8 +455,20 @@ class OmniLoop(BaseLoop): logger.error(f"No JSON found in content") return True, action_screenshot_saved except json.JSONDecodeError as e: - logger.error(f"Failed to parse JSON from text: {str(e)}") - return True, action_screenshot_saved + # Try to sanitize the JSON string and retry + try: + # Remove or replace invalid control characters + import re # Local import to ensure availability + + sanitized_text = re.sub(r"[\x00-\x1F\x7F]", "", raw_text) + # Try parsing again with sanitized text + parsed_content = json.loads(sanitized_text) + logger.info( + "Successfully parsed JSON after sanitizing control characters" + ) + except json.JSONDecodeError: + logger.error(f"Failed to parse JSON from text: {str(e)}") + return True, action_screenshot_saved # Step 4: Process the parsed content if available if parsed_content: diff --git a/libs/agent/agent/ui/gradio/app.py b/libs/agent/agent/ui/gradio/app.py index 4fb4b4fd..44027317 100644 --- a/libs/agent/agent/ui/gradio/app.py +++ b/libs/agent/agent/ui/gradio/app.py @@ -271,16 +271,19 @@ def create_agent( api_key = os.environ.get("ANTHROPIC_API_KEY", "") # Create LLM model object with appropriate parameters - provider_base_url = "http://localhost:8000/v1" if use_oaicompat else None + provider_base_url = "http://localhost:1234/v1" if use_oaicompat else None if use_oaicompat: - # Special handling for OAICOMPAT - use OPENAI provider with custom base URL - print(f"DEBUG - Creating OAICOMPAT agent with model: {model_name}") + # Special handling for OAICOMPAT - use OAICOMPAT provider with custom base URL + print( + f"DEBUG - Creating OAICOMPAT agent with model: {model_name}, URL: {provider_base_url}" + ) llm = LLM( - provider=provider, # Already set to OPENAI + provider=LLMProvider.OAICOMPAT, # Set to OAICOMPAT instead of using original provider name=model_name, provider_base_url=provider_base_url, ) + print(f"DEBUG - LLM provider is now: {llm.provider}, base URL: {llm.provider_base_url}") # Note: Don't pass use_oaicompat to the agent, as it doesn't accept this parameter elif provider == LLMProvider.OAICOMPAT: # This path is unlikely to be taken with our current approach @@ -461,8 +464,10 @@ def respond( # Special handling for OAICOMPAT to bypass provider-specific errors # Creates the agent with OPENAI provider but using custom model name and provider base URL is_oaicompat = str(provider) == "oaicompat" - if is_oaicompat: - provider = LLMProvider.OPENAI + + # Don't override the provider for OAICOMPAT - instead pass it through + # if is_oaicompat: + # provider = LLMProvider.OPENAI # Get API key based on provider if provider == LLMProvider.OPENAI: