mirror of
https://github.com/trycua/computer.git
synced 2026-01-06 21:39:58 -06:00
First pass on new docs for Lume, Computer-Server, Core, Lumier, MCP-Server, SOM. Add support for mermaid diagrams
This commit is contained in:
Morgan Dean
@@ -2,206 +2,4 @@
|
||||
title: Lume
|
||||
---
|
||||
|
||||
Lume exposes a local HTTP API server that listens on `http://localhost:7777/lume`, enabling automated management of VMs.
|
||||
|
||||
## Base URL
|
||||
|
||||
```
|
||||
http://localhost:7777/lume
|
||||
```
|
||||
|
||||
## Endpoints
|
||||
|
||||
<details open>
|
||||
<summary><strong>Create VM</strong> - POST /vms</summary>
|
||||
|
||||
```bash
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name": "lume_vm",
|
||||
"os": "macOS",
|
||||
"cpu": 2,
|
||||
"memory": "4GB",
|
||||
"diskSize": "64GB",
|
||||
"display": "1024x768",
|
||||
"ipsw": "latest",
|
||||
"storage": "ssd"
|
||||
}' \
|
||||
http://localhost:7777/lume/vms
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details open>
|
||||
<summary><strong>Run VM</strong> - POST /vms/:name/run</summary>
|
||||
|
||||
```bash
|
||||
# Basic run
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X POST \
|
||||
http://localhost:7777/lume/vms/my-vm-name/run
|
||||
|
||||
# Run with VNC client started and shared directory
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"noDisplay": false,
|
||||
"sharedDirectories": [
|
||||
{
|
||||
"hostPath": "~/Projects",
|
||||
"readOnly": false
|
||||
}
|
||||
],
|
||||
"recoveryMode": false,
|
||||
"storage": "ssd"
|
||||
}' \
|
||||
http://localhost:7777/lume/vms/lume_vm/run
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details open>
|
||||
<summary><strong>List VMs</strong> - GET /vms</summary>
|
||||
|
||||
```bash
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
http://localhost:7777/lume/vms
|
||||
```
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"name": "my-vm",
|
||||
"state": "stopped",
|
||||
"os": "macOS",
|
||||
"cpu": 2,
|
||||
"memory": "4GB",
|
||||
"diskSize": "64GB"
|
||||
},
|
||||
{
|
||||
"name": "my-vm-2",
|
||||
"state": "stopped",
|
||||
"os": "linux",
|
||||
"cpu": 2,
|
||||
"memory": "4GB",
|
||||
"diskSize": "64GB"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details open>
|
||||
<summary><strong>Get VM Details</strong> - GET /vms/:name</summary>
|
||||
|
||||
```bash
|
||||
# Basic get
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
http://localhost:7777/lume/vms/lume_vm
|
||||
|
||||
# Get with specific storage
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
http://localhost:7777/lume/vms/lume_vm?storage=ssd
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "lume_vm",
|
||||
"state": "stopped",
|
||||
"os": "macOS",
|
||||
"cpu": 2,
|
||||
"memory": "4GB",
|
||||
"diskSize": "64GB",
|
||||
"display": "1024x768",
|
||||
"ipAddress": "192.168.65.2",
|
||||
"vncPort": 5900,
|
||||
"sharedDirectories": [
|
||||
{
|
||||
"hostPath": "~/Projects",
|
||||
"readOnly": false,
|
||||
"tag": "com.apple.virtio-fs.automount"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details open>
|
||||
<summary><strong>Update VM Configuration</strong> - PUT /vms/:name</summary>
|
||||
|
||||
```bash
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X PUT \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"cpu": 4,
|
||||
"memory": "8GB",
|
||||
"diskSize": "100GB",
|
||||
"display": "1920x1080",
|
||||
"storage": "ssd"
|
||||
}' \
|
||||
http://localhost:7777/lume/vms/lume_vm
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details open>
|
||||
<summary><strong>Stop VM</strong> - POST /vms/:name/stop</summary>
|
||||
|
||||
```bash
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X POST \
|
||||
http://localhost:7777/lume/vms/lume_vm/stop
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details open>
|
||||
<summary><strong>Delete VM</strong> - DELETE /vms/:name</summary>
|
||||
|
||||
```bash
|
||||
# Basic delete
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X DELETE \
|
||||
http://localhost:7777/lume/vms/lume_vm
|
||||
|
||||
# Delete with specific storage
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X DELETE \
|
||||
http://localhost:7777/lume/vms/lume_vm?storage=ssd
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details open>
|
||||
<summary><strong>Pull VM Image</strong> - POST /images/pull</summary>
|
||||
|
||||
```bash
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"image": "macos-sequoia-vanilla:latest",
|
||||
"registry": "ghcr.io",
|
||||
"organization": "trycua",
|
||||
"storage": "ssd"
|
||||
}' \
|
||||
http://localhost:7777/lume/images/pull
|
||||
```
|
||||
|
||||
</details>
|
||||
Coming soon
|
||||
|
||||
@@ -10,4 +10,38 @@ The SOM library provides visual element detection and interaction capabilities.
|
||||
|
||||
## API Documentation
|
||||
|
||||
Coming soon.
|
||||
### OmniParser Class
|
||||
|
||||
```python
|
||||
class OmniParser:
|
||||
def __init__(self, device: str = "auto"):
|
||||
"""Initialize the parser with automatic device detection"""
|
||||
|
||||
def parse(
|
||||
self,
|
||||
image: PIL.Image,
|
||||
box_threshold: float = 0.3,
|
||||
iou_threshold: float = 0.1,
|
||||
use_ocr: bool = True,
|
||||
ocr_engine: str = "easyocr"
|
||||
) -> ParseResult:
|
||||
"""Parse UI elements from an image"""
|
||||
```
|
||||
|
||||
### ParseResult Object
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class ParseResult:
|
||||
elements: List[UIElement] # Detected elements
|
||||
visualized_image: PIL.Image # Annotated image
|
||||
processing_time: float # Time in seconds
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to JSON-serializable dictionary"""
|
||||
|
||||
def filter_by_type(self, elem_type: str) -> List[UIElement]:
|
||||
"""Filter elements by type ('icon' or 'text')"""
|
||||
```
|
||||
|
||||
More Coming soon.
|
||||
|
||||
@@ -241,8 +241,11 @@ async for result in agent.run(task):
|
||||
---
|
||||
|
||||
<Callout type="info">
|
||||
**Need detailed API documentation?** Explore the complete API reference with
|
||||
detailed class documentation, and method signatures.
|
||||
**Need detailed API documentation?**{' '}
|
||||
<p>
|
||||
Explore the complete API reference with detailed class documentation, and
|
||||
method signatures.
|
||||
</p>
|
||||
<a
|
||||
href="/api/agent"
|
||||
className={cn(
|
||||
@@ -252,6 +255,6 @@ async for result in agent.run(task):
|
||||
'no-underline h-10'
|
||||
)}>
|
||||
View API Reference
|
||||
<ChevronRight />
|
||||
<ChevronRight size={18} />
|
||||
</a>
|
||||
</Callout>
|
||||
|
||||
@@ -1,61 +1,55 @@
|
||||
---
|
||||
title: Computer Server
|
||||
description: The server component for the Computer-Use Interface framework.
|
||||
pypi: cua-computer-server
|
||||
macos: true
|
||||
linux: true
|
||||
windows: true
|
||||
---
|
||||
|
||||
<div
|
||||
align="center"
|
||||
style={{
|
||||
display: 'flex',
|
||||
gap: '10px',
|
||||
margin: '0 auto',
|
||||
width: '100%',
|
||||
justifyContent: 'center',
|
||||
}}
|
||||
>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Python-333333?logo=python&logoColor=white&labelColor=blue"
|
||||
alt="Python"
|
||||
/>
|
||||
</a>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=F0F0F0"
|
||||
alt="macOS"
|
||||
/>
|
||||
</a>
|
||||
<a href="https://discord.com/invite/mVnXXpdE85">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Discord-%235865F2.svg?&logo=discord&logoColor=white"
|
||||
alt="Discord"
|
||||
/>
|
||||
</a>
|
||||
<a href="https://pypi.org/project/cua-computer-server/">
|
||||
<img
|
||||
src="https://img.shields.io/pypi/v/cua-computer-server?color=blue"
|
||||
alt="PyPI"
|
||||
/>
|
||||
</a>
|
||||
</div>
|
||||
import { buttonVariants } from 'fumadocs-ui/components/ui/button';
|
||||
import { cn } from 'fumadocs-ui/utils/cn';
|
||||
import { ChevronRight } from 'lucide-react';
|
||||
|
||||
**Computer Server** is the server component for the Computer-Use Interface (CUI) framework powering Cua for interacting with local macOS and Linux sandboxes, PyAutoGUI-compatible, and pluggable with any AI agent systems (Cua, Langchain, CrewAI, AutoGen).
|
||||
**Computer Server** provides the websocket interface for the [Computer-Use Interface (CUI)](./computer/) to interact with.
|
||||
|
||||
## Features
|
||||
|
||||
- WebSocket API for computer-use
|
||||
- Cross-platform support (macOS, Linux)
|
||||
- Integration with CUA computer library for screen control, keyboard/mouse automation, and accessibility
|
||||
- Cross-platform support (macOS, Linux, Windows)
|
||||
- Integration with the CUI library for screen control, keyboard/mouse automation, and accessibility
|
||||
|
||||
## Install
|
||||
|
||||
To install the Computer-Use Interface (CUI):
|
||||
|
||||
```bash
|
||||
pip install cua-computer-server
|
||||
```
|
||||
|
||||
## Run
|
||||
## Examples & Guides
|
||||
|
||||
Refer to this notebook for a step-by-step guide on how to use the Computer-Use Server on the host system or VM:
|
||||
<Cards>
|
||||
<Card
|
||||
href="https://github.com/trycua/cua/tree/main/notebooks/computer_server_nb.ipynb"
|
||||
title="Computer-Use Server Notebook">
|
||||
Step-by-step guide using the Computer-Use Server on a host system or virtual
|
||||
machine.
|
||||
</Card>
|
||||
</Cards>
|
||||
|
||||
- [Computer-Use Server](https://github.com/trycua/cua/tree/main/notebooks/samples/computer_server_nb.ipynb)
|
||||
---
|
||||
|
||||
<Callout type="info">
|
||||
**Need detailed API documentation?** Explore the complete API reference with
|
||||
detailed class documentation, and method signatures.
|
||||
<a
|
||||
href="/api/computer-server"
|
||||
className={cn(
|
||||
buttonVariants({
|
||||
color: 'secondary',
|
||||
}),
|
||||
'no-underline h-10'
|
||||
)}>
|
||||
View API Reference
|
||||
<ChevronRight size={18} />
|
||||
</a>
|
||||
</Callout>
|
||||
|
||||
@@ -13,7 +13,9 @@ import { buttonVariants } from 'fumadocs-ui/components/ui/button';
|
||||
import { cn } from 'fumadocs-ui/utils/cn';
|
||||
import { ChevronRight } from 'lucide-react';
|
||||
|
||||
Computer powers Cua systems and is PyAutoGUI-compatible and pluggable with any AI agent system (Cua, Langchain, CrewAI, AutoGen). It relies on [Lume](./lume.mdx) for creating and managing sandbox environments.
|
||||
Computer, when paired with [Computer Server](../computer-server.mdx) enables programatic interaction with cross-platform sandboxes. It powers C/ua systems and is PyAutoGUI-compatible and pluggable with any AI agent system (Cua, Langchain, CrewAI, AutoGen).
|
||||
|
||||
The Python version relies on [Lume](./lume.mdx) for creating and managing sandbox environments.
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -32,7 +34,7 @@ Computer powers Cua systems and is PyAutoGUI-compatible and pluggable with any A
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
## Key Features
|
||||
## Features
|
||||
|
||||
- Create and manage virtual machine sandboxes
|
||||
- Take screenshots of the virtual machine
|
||||
@@ -161,8 +163,11 @@ Computer powers Cua systems and is PyAutoGUI-compatible and pluggable with any A
|
||||
---
|
||||
|
||||
<Callout type="info">
|
||||
**Need detailed API documentation?** Explore the complete API reference with
|
||||
detailed class documentation, and method signatures.
|
||||
**Need detailed API documentation?**{' '}
|
||||
<p>
|
||||
Explore the complete API reference with detailed class documentation, and
|
||||
method signatures.
|
||||
</p>
|
||||
<a
|
||||
href="/api/computer"
|
||||
className={cn(
|
||||
@@ -172,6 +177,6 @@ Computer powers Cua systems and is PyAutoGUI-compatible and pluggable with any A
|
||||
'no-underline h-10'
|
||||
)}>
|
||||
View API Reference
|
||||
<ChevronRight />
|
||||
<ChevronRight size={18} />
|
||||
</a>
|
||||
</Callout>
|
||||
|
||||
@@ -1,44 +1,18 @@
|
||||
---
|
||||
title: Core
|
||||
description: Core infrastructure and shared utilities powering the Cua computer-use platform
|
||||
pypi: cua-core
|
||||
npm: '@trycua/core'
|
||||
macos: true
|
||||
windows: true
|
||||
linux: true
|
||||
---
|
||||
|
||||
<div
|
||||
align="center"
|
||||
style={{
|
||||
display: "flex",
|
||||
gap: "10px",
|
||||
margin: "0 auto",
|
||||
width: "100%",
|
||||
justifyContent: "center",
|
||||
}}
|
||||
>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Python-333333?logo=python&logoColor=white&labelColor=blue"
|
||||
alt="Python"
|
||||
/>
|
||||
</a>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=F0F0F0"
|
||||
alt="macOS"
|
||||
/>
|
||||
</a>
|
||||
<a href="https://discord.com/invite/mVnXXpdE85">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Discord-%235865F2.svg?&logo=discord&logoColor=white"
|
||||
alt="Discord"
|
||||
/>
|
||||
</a>
|
||||
<a href="https://pypi.org/project/cua-computer-server/">
|
||||
<img
|
||||
src="https://img.shields.io/pypi/v/cua-computer-server?color=blue"
|
||||
alt="PyPI"
|
||||
/>
|
||||
</a>
|
||||
</div>
|
||||
import { buttonVariants } from 'fumadocs-ui/components/ui/button';
|
||||
import { cn } from 'fumadocs-ui/utils/cn';
|
||||
import { ChevronRight } from 'lucide-react';
|
||||
|
||||
**Cua Core** provides essential shared functionality and utilities used across the Cua ecosystem:
|
||||
# Features
|
||||
|
||||
- Privacy-focused telemetry system for transparent usage analytics
|
||||
- Common helper functions and utilities used by other Cua packages
|
||||
@@ -49,3 +23,24 @@ title: Core
|
||||
```bash
|
||||
pip install cua-core
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
<Callout type="info">
|
||||
**Need detailed API documentation?**{' '}
|
||||
<p>
|
||||
Explore the complete API reference with detailed class documentation, and
|
||||
method signatures.
|
||||
</p>
|
||||
<a
|
||||
href="/api/core"
|
||||
className={cn(
|
||||
buttonVariants({
|
||||
color: 'secondary',
|
||||
}),
|
||||
'no-underline h-10'
|
||||
)}>
|
||||
View API Reference
|
||||
<ChevronRight size={18} />
|
||||
</a>
|
||||
</Callout>
|
||||
|
||||
@@ -6,3 +6,14 @@ description: Getting started with the C/ua libraries
|
||||
## Overview
|
||||
|
||||
The C/ua project provides several libraries for building Computer-Use AI agents.
|
||||
|
||||
| Library | Description | Installation |
|
||||
| -------------------------------------------- | -------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- |
|
||||
| [**Lume**](./lume.mdx) | VM management for macOS/Linux using Apple's Virtualization.Framework | `curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh \| bash` |
|
||||
| [**Lumier**](./lumier.mdx) | Docker interface for macOS and Linux VMs | `docker pull trycua/lumier:latest` |
|
||||
| [**Computer**](./computer.mdx) | Python Interface for controlling virtual machines | `pip install "cua-computer[all]"`<br/><br/>`npm install @trycua/computer` |
|
||||
| [**Agent**](./agent.mdx) | AI agent framework for automating tasks | `pip install "cua-agent[all]"` |
|
||||
| [**MCP Server**](./mcp-server.mdx) | MCP server for using CUA with Claude Desktop | `pip install cua-mcp-server` |
|
||||
| [**SOM**](./som.mdx) | Self-of-Mark library for Agent | `pip install cua-som` |
|
||||
| [**Computer Server**](./computer-server.mdx) | Server component for Computer | `pip install cua-computer-server` |
|
||||
| [**Core**](./core.mdx) | Python Core utilities | `pip install cua-core`<br/><br/>`npm install @trycua/core` |
|
||||
|
||||
592
docs/content/docs/home/libraries/lume/http-api.mdx
Normal file
592
docs/content/docs/home/libraries/lume/http-api.mdx
Normal file
@@ -0,0 +1,592 @@
|
||||
---
|
||||
title: HTTP Server API
|
||||
description: Lume exposes a local HTTP API server that listens at localhost for programatic management of VMs.
|
||||
---
|
||||
|
||||
import { Tabs, Tab } from 'fumadocs-ui/components/tabs';
|
||||
|
||||
## Default URL
|
||||
|
||||
```
|
||||
http://localhost:7777
|
||||
```
|
||||
|
||||
## Endpoints
|
||||
|
||||
### Create VM
|
||||
|
||||
Create a new virtual machine.
|
||||
|
||||
`POST: /vms`
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Name | Type | Required | Description |
|
||||
| -------- | ------- | -------- | ------------------------------------ |
|
||||
| name | string | Yes | Name of the VM |
|
||||
| os | string | Yes | Guest OS (`macOS`, `linux`, etc.) |
|
||||
| cpu | integer | Yes | Number of CPU cores |
|
||||
| memory | string | Yes | Memory size (e.g. `4GB`) |
|
||||
| diskSize | string | Yes | Disk size (e.g. `64GB`) |
|
||||
| display | string | No | Display resolution (e.g. `1024x768`) |
|
||||
| ipsw | string | No | IPSW version (e.g. `latest`) |
|
||||
| storage | string | No | Storage type (`ssd`, etc.) |
|
||||
|
||||
#### Example Request
|
||||
|
||||
<Tabs groupId="language" persist items={['Curl', 'Python', 'TypeScript']}>
|
||||
<Tab value="Curl">
|
||||
|
||||
```bash
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name": "lume_vm",
|
||||
"os": "macOS",
|
||||
"cpu": 2,
|
||||
"memory": "4GB",
|
||||
"diskSize": "64GB",
|
||||
"display": "1024x768",
|
||||
"ipsw": "latest",
|
||||
"storage": "ssd"
|
||||
}' \
|
||||
http://localhost:7777/lume/vms
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Python">
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
payload = {
|
||||
"name": "lume_vm",
|
||||
"os": "macOS",
|
||||
"cpu": 2,
|
||||
"memory": "4GB",
|
||||
"diskSize": "64GB",
|
||||
"display": "1024x768",
|
||||
"ipsw": "latest",
|
||||
"storage": "ssd"
|
||||
}
|
||||
r = requests.post("http://localhost:7777/lume/vms", json=payload, timeout=50)
|
||||
print(r.json())
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="TypeScript">
|
||||
|
||||
```typescript
|
||||
const payload = {
|
||||
name: "lume_vm",
|
||||
os: "macOS",
|
||||
cpu: 2,
|
||||
memory: "4GB",
|
||||
diskSize: "64GB",
|
||||
display: "1024x768",
|
||||
ipsw: "latest",
|
||||
storage: "ssd"
|
||||
}
|
||||
|
||||
const res = await fetch('http://localhost:7777/lume/vms', {
|
||||
methdo: 'POST'
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
console.log(await res.json())
|
||||
```
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
### Run VM
|
||||
|
||||
Run a virtual machine instance.
|
||||
|
||||
`POST: /vms/:name/run`
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Name | Type | Required | Description |
|
||||
| ----------------- | --------------- | -------- | --------------------------------------------------- |
|
||||
| noDisplay | boolean | No | If true, do not start VNC client |
|
||||
| sharedDirectories | array of object | No | List of shared directories (`hostPath`, `readOnly`) |
|
||||
| recoveryMode | boolean | No | Start in recovery mode |
|
||||
| storage | string | No | Storage type (`ssd`, etc.) |
|
||||
|
||||
#### Example Request
|
||||
|
||||
<Tabs groupId="language" persist items={['Curl', 'Python', 'TypeScript']}>
|
||||
<Tab value="Curl">
|
||||
|
||||
```bash
|
||||
# Basic run
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X POST \
|
||||
http://localhost:7777/lume/vms/my-vm-name/run
|
||||
|
||||
# Run with VNC client started and shared directory
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"noDisplay": false,
|
||||
"sharedDirectories": [
|
||||
{
|
||||
"hostPath": "~/Projects",
|
||||
"readOnly": false
|
||||
}
|
||||
],
|
||||
"recoveryMode": false,
|
||||
"storage": "ssd"
|
||||
}' \
|
||||
http://localhost:7777/lume/vms/lume_vm/run
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Python">
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
# Basic run
|
||||
r = requests.post("http://localhost:7777/lume/vms/my-vm-name/run", timeout=50)
|
||||
print(r.json())
|
||||
|
||||
# With VNC and shared directory
|
||||
payload = {
|
||||
"noDisplay": False,
|
||||
"sharedDirectories": [
|
||||
{"hostPath": "~/Projects", "readOnly": False}
|
||||
],
|
||||
"recoveryMode": False,
|
||||
"storage": "ssd"
|
||||
}
|
||||
r = requests.post("http://localhost:7777/lume/vms/lume_vm/run", json=payload, timeout=50)
|
||||
print(r.json())
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="TypeScript">
|
||||
|
||||
```typescript
|
||||
// Basic run
|
||||
const res = await fetch('http://localhost:7777/lume/vms/my-vm-name/run', {
|
||||
method: 'POST',
|
||||
});
|
||||
console.log(await res.json());
|
||||
|
||||
// With VNC and shared directory
|
||||
const payload = {
|
||||
noDisplay: false,
|
||||
sharedDirectories: [{ hostPath: '~/Projects', readOnly: false }],
|
||||
recoveryMode: false,
|
||||
storage: 'ssd',
|
||||
};
|
||||
const res2 = await fetch('http://localhost:7777/lume/vms/lume_vm/run', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
console.log(await res2.json());
|
||||
```
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
### List VMs
|
||||
|
||||
List all virtual machines.
|
||||
|
||||
`GET: /vms`
|
||||
|
||||
#### Example Request
|
||||
|
||||
<Tabs groupId="language" persist items={['Curl', 'Python', 'TypeScript']}>
|
||||
<Tab value="Curl">
|
||||
|
||||
```bash
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
http://localhost:7777/lume/vms
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Python">
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
r = requests.get("http://localhost:7777/lume/vms", timeout=50)
|
||||
print(r.json())
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="TypeScript">
|
||||
|
||||
```typescript
|
||||
const res = await fetch('http://localhost:7777/lume/vms');
|
||||
console.log(await res.json());
|
||||
```
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"name": "my-vm",
|
||||
"state": "stopped",
|
||||
"os": "macOS",
|
||||
"cpu": 2,
|
||||
"memory": "4GB",
|
||||
"diskSize": "64GB"
|
||||
},
|
||||
{
|
||||
"name": "my-vm-2",
|
||||
"state": "stopped",
|
||||
"os": "linux",
|
||||
"cpu": 2,
|
||||
"memory": "4GB",
|
||||
"diskSize": "64GB"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### Get VM Details
|
||||
|
||||
Get details for a specific virtual machine.
|
||||
|
||||
`GET: /vms/:name`
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Name | Type | Required | Description |
|
||||
| ------- | ------ | -------- | -------------------------- |
|
||||
| storage | string | No | Storage type (`ssd`, etc.) |
|
||||
|
||||
#### Example Request
|
||||
|
||||
<Tabs groupId="language" persist items={['Curl', 'Python', 'TypeScript']}>
|
||||
<Tab value="Curl">
|
||||
|
||||
```bash
|
||||
# Basic get
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
http://localhost:7777/lume/vms/lume_vm
|
||||
|
||||
# Get with specific storage
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
http://localhost:7777/lume/vms/lume_vm?storage=ssd
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Python">
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
# Basic get
|
||||
details = requests.get("http://localhost:7777/lume/vms/lume_vm", timeout=50)
|
||||
print(details.json())
|
||||
|
||||
# Get with specific storage
|
||||
details = requests.get("http://localhost:7777/lume/vms/lume_vm", params={"storage": "ssd"}, timeout=50)
|
||||
print(details.json())
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="TypeScript">
|
||||
|
||||
```typescript
|
||||
// Basic get
|
||||
const res = await fetch('http://localhost:7777/lume/vms/lume_vm');
|
||||
console.log(await res.json());
|
||||
|
||||
// Get with specific storage
|
||||
const res2 = await fetch('http://localhost:7777/lume/vms/lume_vm?storage=ssd');
|
||||
console.log(await res2.json());
|
||||
```
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "lume_vm",
|
||||
"state": "stopped",
|
||||
"os": "macOS",
|
||||
"cpu": 2,
|
||||
"memory": "4GB",
|
||||
"diskSize": "64GB",
|
||||
"display": "1024x768",
|
||||
"ipAddress": "192.168.65.2",
|
||||
"vncPort": 5900,
|
||||
"sharedDirectories": [
|
||||
{
|
||||
"hostPath": "~/Projects",
|
||||
"readOnly": false,
|
||||
"tag": "com.apple.virtio-fs.automount"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Update VM Configuration
|
||||
|
||||
Update the configuration of a virtual machine.
|
||||
|
||||
`PUT: /vms/:name`
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Name | Type | Required | Description |
|
||||
| -------- | ------- | -------- | ------------------------------------- |
|
||||
| cpu | integer | No | Number of CPU cores |
|
||||
| memory | string | No | Memory size (e.g. `8GB`) |
|
||||
| diskSize | string | No | Disk size (e.g. `100GB`) |
|
||||
| display | string | No | Display resolution (e.g. `1920x1080`) |
|
||||
| storage | string | No | Storage type (`ssd`, etc.) |
|
||||
|
||||
#### Example Request
|
||||
|
||||
<Tabs groupId="language" persist items={['Curl', 'Python', 'TypeScript']}>
|
||||
<Tab value="Curl">
|
||||
|
||||
```bash
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X PUT \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"cpu": 4,
|
||||
"memory": "8GB",
|
||||
"diskSize": "100GB",
|
||||
"display": "1920x1080",
|
||||
"storage": "ssd"
|
||||
}' \
|
||||
http://localhost:7777/lume/vms/lume_vm
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Python">
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
payload = {
|
||||
"cpu": 4,
|
||||
"memory": "8GB",
|
||||
"diskSize": "100GB",
|
||||
"display": "1920x1080",
|
||||
"storage": "ssd"
|
||||
}
|
||||
r = requests.put("http://localhost:7777/lume/vms/lume_vm", json=payload, timeout=50)
|
||||
print(r.json())
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="TypeScript">
|
||||
|
||||
```typescript
|
||||
const payload = {
|
||||
cpu: 4,
|
||||
memory: '8GB',
|
||||
diskSize: '100GB',
|
||||
display: '1920x1080',
|
||||
storage: 'ssd',
|
||||
};
|
||||
const res = await fetch('http://localhost:7777/lume/vms/lume_vm', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
console.log(await res.json());
|
||||
```
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
### Stop VM
|
||||
|
||||
Stop a running virtual machine.
|
||||
|
||||
`POST: /vms/:name/stop`
|
||||
|
||||
#### Example Request
|
||||
|
||||
<Tabs groupId="language" persist items={['Curl', 'Python', 'TypeScript']}>
|
||||
<Tab value="Curl">
|
||||
|
||||
```bash
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X POST \
|
||||
http://localhost:7777/lume/vms/lume_vm/stop
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Python">
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
r = requests.post("http://localhost:7777/lume/vms/lume_vm/stop", timeout=50)
|
||||
print(r.json())
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="TypeScript">
|
||||
|
||||
```typescript
|
||||
const res = await fetch('http://localhost:7777/lume/vms/lume_vm/stop', {
|
||||
method: 'POST',
|
||||
});
|
||||
console.log(await res.json());
|
||||
```
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
### Delete VM
|
||||
|
||||
Delete a virtual machine instance.
|
||||
|
||||
`DELETE: /vms/:name`
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Name | Type | Required | Description |
|
||||
| ------- | ------ | -------- | -------------------------- |
|
||||
| storage | string | No | Storage type (`ssd`, etc.) |
|
||||
|
||||
#### Example Request
|
||||
|
||||
<Tabs groupId="language" persist items={['Curl', 'Python', 'TypeScript']}>
|
||||
<Tab value="Curl">
|
||||
|
||||
```bash
|
||||
# Basic delete
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X DELETE \
|
||||
http://localhost:7777/lume/vms/lume_vm
|
||||
|
||||
# Delete with specific storage
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X DELETE \
|
||||
http://localhost:7777/lume/vms/lume_vm?storage=ssd
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Python">
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
# Basic delete
|
||||
r = requests.delete("http://localhost:7777/lume/vms/lume_vm", timeout=50)
|
||||
print(r.status_code)
|
||||
|
||||
# Delete with specific storage
|
||||
r = requests.delete("http://localhost:7777/lume/vms/lume_vm", params={"storage": "ssd"}, timeout=50)
|
||||
print(r.status_code)
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="TypeScript">
|
||||
|
||||
```typescript
|
||||
// Basic delete
|
||||
const res = await fetch('http://localhost:7777/lume/vms/lume_vm', {
|
||||
method: 'DELETE',
|
||||
});
|
||||
console.log(res.status);
|
||||
|
||||
// Delete with specific storage
|
||||
const res2 = await fetch('http://localhost:7777/lume/vms/lume_vm?storage=ssd', {
|
||||
method: 'DELETE',
|
||||
});
|
||||
console.log(res2.status);
|
||||
```
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
### Pull VM Image
|
||||
|
||||
Pull a VM image from a registry.
|
||||
|
||||
`POST: /images/pull`
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Name | Type | Required | Description |
|
||||
| ------------ | ------ | -------- | ------------------------------------- |
|
||||
| image | string | Yes | Image name (e.g. `macos-sequoia-...`) |
|
||||
| registry | string | Yes | Registry host (e.g. `ghcr.io`) |
|
||||
| organization | string | Yes | Organization name |
|
||||
| storage | string | No | Storage type (`ssd`, etc.) |
|
||||
|
||||
#### Example Request
|
||||
|
||||
<Tabs groupId="language" persist items={['Curl', 'Python', 'TypeScript']}>
|
||||
<Tab value="Curl">
|
||||
|
||||
```bash
|
||||
curl --connect-timeout 6000 \
|
||||
--max-time 5000 \
|
||||
-X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"image": "macos-sequoia-vanilla:latest",
|
||||
"registry": "ghcr.io",
|
||||
"organization": "trycua",
|
||||
"storage": "ssd"
|
||||
}' \
|
||||
http://localhost:7777/lume/images/pull
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="Python">
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
payload = {
|
||||
"image": "macos-sequoia-vanilla:latest",
|
||||
"registry": "ghcr.io",
|
||||
"organization": "trycua",
|
||||
"storage": "ssd"
|
||||
}
|
||||
r = requests.post("http://localhost:7777/lume/images/pull", json=payload, timeout=50)
|
||||
print(r.json())
|
||||
```
|
||||
|
||||
</Tab>
|
||||
<Tab value="TypeScript">
|
||||
|
||||
```typescript
|
||||
const payload = {
|
||||
image: 'macos-sequoia-vanilla:latest',
|
||||
registry: 'ghcr.io',
|
||||
organization: 'trycua',
|
||||
storage: 'ssd',
|
||||
};
|
||||
const res = await fetch('http://localhost:7777/lume/images/pull', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
console.log(await res.json());
|
||||
```
|
||||
|
||||
</Tab>
|
||||
</Tabs>
|
||||
@@ -1,49 +1,43 @@
|
||||
---
|
||||
title: Lume
|
||||
description: A lightweight Command Line Interface and local API server for creating, running and managing macOS and Linux virtual machines.
|
||||
macos: true
|
||||
linux: true
|
||||
---
|
||||
|
||||
<div
|
||||
align="center"
|
||||
style={{
|
||||
display: 'flex',
|
||||
gap: '10px',
|
||||
margin: '0 auto',
|
||||
width: '100%',
|
||||
justifyContent: 'center',
|
||||
}}
|
||||
>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Swift_6-F54A2A?logo=swift&logoColor=white&labelColor=F54A2A"
|
||||
alt="Swift 6"
|
||||
/>
|
||||
</a>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=F0F0F0"
|
||||
alt="macOS"
|
||||
/>
|
||||
</a>
|
||||
<a href="https://discord.com/invite/mVnXXpdE85">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Discord-%235865F2.svg?&logo=discord&logoColor=white"
|
||||
alt="Discord"
|
||||
/>
|
||||
</a>
|
||||
</div>
|
||||
import Link from 'next/link';
|
||||
import { buttonVariants } from 'fumadocs-ui/components/ui/button';
|
||||
import { Step, Steps } from 'fumadocs-ui/components/steps';
|
||||
import { cn } from 'fumadocs-ui/utils/cn';
|
||||
import { ChevronRight } from 'lucide-react';
|
||||
|
||||
**lume** is a lightweight Command Line Interface and local API server to create, run and manage macOS and Linux virtual machines (VMs) with near-native performance on Apple Silicon, using Apple's `Virtualization.Framework`.
|
||||
# Lume
|
||||
|
||||
### Run prebuilt macOS images in just 1 step
|
||||
Lume is a lightweight Command Line Interface and local API server for creating, running and managing **macOS and Linux virtual machines** with near-native performance on Apple Silicon, using Apple's `Virtualization.Framework`.
|
||||
|
||||
<div align="center">
|
||||
<img src="/img/cli.png" alt="lume cli" />
|
||||
</div>
|
||||
## Quick Start
|
||||
|
||||
Install and run a prebuilt macOS VM in two commands:
|
||||
|
||||
```bash
|
||||
# Install Lume
|
||||
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh)"
|
||||
# Pull & start a macOS image
|
||||
lume run macos-sequoia-vanilla:latest
|
||||
```
|
||||
|
||||
<Callout type="warning">
|
||||
**Security Note**: All prebuilt images use the default password `lume`. Change
|
||||
this immediately after your first login using the `passwd` command.
|
||||
</Callout>
|
||||
|
||||
**System Requirements**:
|
||||
|
||||
- Apple Silicon Mac (M1, M2, M3, etc.)
|
||||
- macOS 13.0 or later
|
||||
- At least 8GB of RAM (16GB recommended)
|
||||
- At least 50GB of free disk space
|
||||
|
||||
## Install
|
||||
|
||||
Install with a single command:
|
||||
@@ -58,10 +52,65 @@ By default, Lume is installed as a background service that starts automatically
|
||||
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh) --no-background-service"
|
||||
```
|
||||
|
||||
**Note:** With this option, you'll need to manually start the Lume API service by running `lume serve` in your terminal whenever you need to use tools or libraries that rely on the Lume API (such as the Computer-Use Agent).
|
||||
<Callout type="info">
|
||||
**Note:** With this option, you'll need to manually start the Lume API service
|
||||
by running `lume serve` in your terminal whenever you need to use tools or
|
||||
libraries that rely on the Lume API (such as the Computer-Use Agent).
|
||||
</Callout>
|
||||
|
||||
You can also download the `lume.pkg.tar.gz` archive from the [latest release](https://github.com/trycua/lume/releases), extract it, and install the package manually.
|
||||
|
||||
## Getting Started
|
||||
|
||||
Once installed, you can start using Lume with these common workflows:
|
||||
|
||||
<Steps>
|
||||
<Step>
|
||||
### Run a Prebuilt VM
|
||||
|
||||
```bash
|
||||
# Run a macOS Sequoia VM
|
||||
lume run macos-sequoia-vanilla:latest
|
||||
|
||||
# Run an Ubuntu VM
|
||||
lume run ubuntu-noble-vanilla:latest
|
||||
```
|
||||
|
||||
</Step>
|
||||
|
||||
<Step>
|
||||
### Create a Custom VM
|
||||
|
||||
```bash
|
||||
# Create a new macOS VM
|
||||
lume create my-macos-vm --cpu 4 --memory 8GB --disk-size 50GB
|
||||
|
||||
# Create a Linux VM
|
||||
lume create my-linux-vm --os linux --cpu 2 --memory 4GB
|
||||
```
|
||||
|
||||
<Callout type="info">
|
||||
**Disk Space**: The actual disk space used by sparse images will be much lower than the logical size listed. You can resize VM disks after creation using `lume set <name> --disk-size <size>`.
|
||||
</Callout>
|
||||
</Step>
|
||||
|
||||
<Step>
|
||||
### Manage Your VMs
|
||||
|
||||
```bash
|
||||
# List all VMs
|
||||
lume ls
|
||||
|
||||
# Get VM details
|
||||
lume get my-vm
|
||||
|
||||
# Stop a running VM
|
||||
lume stop my-vm
|
||||
```
|
||||
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
@@ -163,16 +212,12 @@ Command Options:
|
||||
--port <port> Port to listen on (default: 7777)
|
||||
```
|
||||
|
||||
## Prebuilt Images
|
||||
## Examples
|
||||
|
||||
### Prebuilt Images
|
||||
|
||||
Pre-built images are available in the registry [ghcr.io/trycua](https://github.com/orgs/trycua/packages).
|
||||
|
||||
**Important Note (v0.2.0+):** Images are being re-uploaded with sparse file system optimizations enabled, resulting in significantly lower actual disk usage. Older images (without the `-sparse` suffix) are now **deprecated**. The last version of `lume` fully supporting the non-sparse images was `v0.1.x`. Starting from `v0.2.0`, lume will automatically pull images optimized with sparse file system support.
|
||||
|
||||
These images come with an SSH server pre-configured and auto-login enabled.
|
||||
|
||||
For the security of your VM, change the default password `lume` immediately after your first login.
|
||||
|
||||
| Image | Tag | Description | Logical Size |
|
||||
| ----------------------- | ------------------- | ----------------------------------------------------------------------------------------------- | ------------ |
|
||||
| `macos-sequoia-vanilla` | `latest`, `15.2` | macOS Sequoia 15.2 image | 20GB |
|
||||
@@ -180,17 +225,65 @@ For the security of your VM, change the default password `lume` immediately afte
|
||||
| `macos-sequoia-cua` | `latest`, `15.3` | macOS Sequoia 15.3 image compatible with the Computer interface | 24GB |
|
||||
| `ubuntu-noble-vanilla` | `latest`, `24.04.1` | [Ubuntu Server for ARM 24.04.1 LTS](https://ubuntu.com/download/server/arm) with Ubuntu Desktop | 20GB |
|
||||
|
||||
For additional disk space, resize the VM disk after pulling the image using the `lume set <name> --disk-size <size>` command. Note that the actual disk space used by sparse images will be much lower than the logical size listed.
|
||||
### Common Workflows
|
||||
|
||||
#### Development Environment Setup
|
||||
|
||||
```bash
|
||||
# Create a development VM with more resources
|
||||
lume create dev-vm --cpu 6 --memory 12GB --disk-size 100GB
|
||||
|
||||
# Run with shared directory for code
|
||||
lume run dev-vm --shared-dir ~/Projects:rw
|
||||
```
|
||||
|
||||
#### Testing Different macOS Versions
|
||||
|
||||
```bash
|
||||
# Pull and run different macOS versions
|
||||
lume pull macos-sequoia-vanilla:latest
|
||||
lume run macos-sequoia-vanilla:latest
|
||||
|
||||
# Clone a VM for testing
|
||||
lume clone my-vm my-vm-test
|
||||
```
|
||||
|
||||
#### File Sharing Examples
|
||||
|
||||
```bash
|
||||
# Share a read-only directory
|
||||
lume run my-vm --shared-dir ~/Documents:ro
|
||||
|
||||
# Share multiple directories
|
||||
lume run my-vm --shared-dir ~/Projects:rw --shared-dir ~/Downloads:ro
|
||||
|
||||
# For Linux VMs, mount additional disk images
|
||||
lume run ubuntu-vm --mount ~/disk-image.img
|
||||
```
|
||||
|
||||
## Local API Server
|
||||
|
||||
`lume` exposes a local HTTP API server that listens on `http://localhost:7777/lume`, enabling automated management of VMs.
|
||||
Lume exposes a local HTTP API server for programmatic VM management, perfect for automation and integration with other tools.
|
||||
|
||||
```bash
|
||||
# Start the API server
|
||||
lume serve
|
||||
```
|
||||
|
||||
For detailed API documentation, see the [API Reference](/v1/api/lume).
|
||||
<Callout type="info">
|
||||
<p>Read the doucmentation on the API server endpoints.</p>
|
||||
<Link
|
||||
href="/home/libraries/lume/http-api"
|
||||
className={cn(
|
||||
buttonVariants({
|
||||
color: 'secondary',
|
||||
}),
|
||||
'no-underline h-10'
|
||||
)}>
|
||||
Lume API Documentation
|
||||
<ChevronRight size={18} />
|
||||
</Link>
|
||||
</Callout>
|
||||
|
||||
## Development
|
||||
|
||||
@@ -205,21 +298,6 @@ This workspace is preconfigured with Swift language support, build tasks, and de
|
||||
|
||||
## FAQ
|
||||
|
||||
### What is Lume?
|
||||
|
||||
Lume is a lightweight Command Line Interface and local API server to create, run and manage macOS and Linux virtual machines (VMs) with near-native performance on Apple Silicon, using Apple's `Virtualization.Framework`.
|
||||
|
||||
### What are the system requirements?
|
||||
|
||||
- Apple Silicon Mac (M1, M2, M3, etc.)
|
||||
- macOS 13.0 or later
|
||||
- At least 8GB of RAM (16GB recommended)
|
||||
- At least 50GB of free disk space
|
||||
|
||||
### How do I change the default password for VMs?
|
||||
|
||||
The default password for all prebuilt images is `lume`. For security, change it immediately after your first login using the `passwd` command.
|
||||
|
||||
### Can I run multiple VMs simultaneously?
|
||||
|
||||
Yes, you can run multiple VMs at the same time as long as your system has sufficient resources (CPU, memory, and disk space).
|
||||
@@ -238,14 +316,6 @@ The shared directory will be automatically mounted in the VM.
|
||||
|
||||
By default, VMs are stored in `~/.lume/vms/`. You can configure additional storage locations using the `lume config storage` commands.
|
||||
|
||||
### How do I access the VM's display?
|
||||
|
||||
Lume automatically starts a VNC client when you run a VM. You can also connect manually using any VNC client to the port shown in the VM details.
|
||||
|
||||
### Can I use Lume programmatically?
|
||||
|
||||
Yes, Lume provides a local HTTP API server. Start it with `lume serve` and access it at `http://localhost:7777/lume`.
|
||||
|
||||
### How do I update Lume?
|
||||
|
||||
Run the install script again to update to the latest version:
|
||||
@@ -254,16 +324,23 @@ Run the install script again to update to the latest version:
|
||||
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh)"
|
||||
```
|
||||
|
||||
## Contributing
|
||||
---
|
||||
|
||||
We welcome and greatly appreciate contributions to lume! Whether you're improving documentation, adding new features, fixing bugs, or adding new VM images, your efforts help make lume better for everyone. For detailed instructions on how to contribute, please refer to our [Contributing Guidelines](CONTRIBUTING.md).
|
||||
|
||||
Join our [Discord community](https://discord.com/invite/mVnXXpdE85) to discuss ideas or get assistance.
|
||||
|
||||
## License
|
||||
|
||||
lume is open-sourced under the MIT License - see the [LICENSE](https://github.com/trycua/cua/blob/main/LICENSE.md) file for details.
|
||||
|
||||
## Trademarks
|
||||
|
||||
Apple, macOS, and Apple Silicon are trademarks of Apple Inc. Ubuntu and Canonical are registered trademarks of Canonical Ltd. This project is not affiliated with, endorsed by, or sponsored by Apple Inc. or Canonical Ltd.
|
||||
<Callout type="info">
|
||||
**Need detailed API documentation?**{' '}
|
||||
<p>
|
||||
Explore the complete API reference with detailed class documentation, and
|
||||
method signatures.
|
||||
</p>
|
||||
<a
|
||||
href="/api/lume"
|
||||
className={cn(
|
||||
buttonVariants({
|
||||
color: 'secondary',
|
||||
}),
|
||||
'no-underline h-10'
|
||||
)}>
|
||||
View API Reference
|
||||
<ChevronRight size={18} />
|
||||
</a>
|
||||
</Callout>
|
||||
@@ -1,85 +1,114 @@
|
||||
---
|
||||
title: Lumier
|
||||
description: Run macOS and Linux virtual machines effortlessly in Docker containers with browser-based VNC access.
|
||||
macos: true
|
||||
linux: true
|
||||
---
|
||||
|
||||
<div
|
||||
align="center"
|
||||
style={{
|
||||
display: 'flex',
|
||||
gap: '10px',
|
||||
margin: '0 auto',
|
||||
width: '100%',
|
||||
justifyContent: 'center',
|
||||
}}
|
||||
>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Swift_6-F54A2A?logo=swift&logoColor=white&labelColor=F54A2A"
|
||||
alt="Swift 6"
|
||||
/>
|
||||
</a>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=F0F0F0"
|
||||
alt="macOS"
|
||||
/>
|
||||
</a>
|
||||
<a href="https://discord.com/invite/mVnXXpdE85">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Discord-%235865F2.svg?&logo=discord&logoColor=white"
|
||||
alt="Discord"
|
||||
/>
|
||||
</a>
|
||||
</div>
|
||||
import { buttonVariants } from 'fumadocs-ui/components/ui/button';
|
||||
import { cn } from 'fumadocs-ui/utils/cn';
|
||||
import { ChevronRight } from 'lucide-react';
|
||||
import Link from 'next/link';
|
||||
import { Step, Steps } from 'fumadocs-ui/components/steps';
|
||||
|
||||
macOS and Linux virtual machines in a Docker container.
|
||||
## What is Lumier?
|
||||
|
||||
Lumier is a streamlined interface for running macOS and Linux virtual machines with minimal setup. It packages a pre-configured environment in Docker that connects to the `lume` virtualization service on your host machine.
|
||||
|
||||
<div align="center">
|
||||
<video
|
||||
src="https://github.com/user-attachments/assets/2ecca01c-cb6f-4c35-a5a7-69bc58bd94e2"
|
||||
width="800"
|
||||
controls
|
||||
></video>
|
||||
controls></video>
|
||||
</div>
|
||||
|
||||
## What is Lumier?
|
||||
### Features
|
||||
|
||||
**Lumier** is an interface for running macOS virtual machines with minimal setup. It uses Docker as a packaging system to deliver a pre-configured environment that connects to the `lume` virtualization service running on your host machine. With Lumier, you get:
|
||||
- **Quick Setup** - Get a VM running in minutes
|
||||
- **Browser Access** - VNC interface accessible from any browser
|
||||
- **Easy File Sharing** - Seamless file transfer between host and VM
|
||||
- **Simple Configuration** - Environment variables for easy customization
|
||||
- **Hardware Acceleration** - Native virtualization using Apple's framework
|
||||
|
||||
- A ready-to-use macOS or Linux virtual machine in minutes
|
||||
- Browser-based VNC access to your VM
|
||||
- Easy file sharing between your host and VM
|
||||
- Simple configuration through environment variables
|
||||
<Callout type="info">
|
||||
Lumier uses Docker as a packaging system, not for isolation. It creates true
|
||||
virtual machines using Apple's Virtualization Framework through the Lume CLI.
|
||||
</Callout>
|
||||
|
||||
## Requirements
|
||||
## Installation
|
||||
|
||||
Before using Lumier, make sure you have:
|
||||
### Prerequisites
|
||||
|
||||
1. **Docker for Apple Silicon** - download it [here](https://desktop.docker.com/mac/main/arm64/Docker.dmg) and follow the installation instructions.
|
||||
<Steps>
|
||||
<Step>
|
||||
### Install Docker for Apple Silicon
|
||||
|
||||
2. **Lume** - This is the virtualization CLI that powers Lumier. Install it with this command:
|
||||
Download and install [Docker Desktop](https://desktop.docker.com/mac/main/arm64/Docker.dmg) for Mac.
|
||||
|
||||
Make sure Docker is running before proceeding to the next step.
|
||||
|
||||
</Step>
|
||||
|
||||
<Step>
|
||||
### Install Lume Virtualization Service
|
||||
|
||||
Install [Lume](./lume/) with a single command:
|
||||
|
||||
```bash
|
||||
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh)"
|
||||
```
|
||||
|
||||
After installation, Lume runs as a background service and listens on port 7777. This service allows Lumier to create and manage virtual machines. If port 7777 is already in use on your system, you can specify a different port with the `--port` option when running the `install.sh` script.
|
||||
|
||||
## How It Works
|
||||
|
||||
> **Note:** We're using Docker primarily as a convenient delivery mechanism, not as an isolation layer. Unlike traditional Docker containers, Lumier leverages the Apple Virtualization Framework (Apple Vz) through the `lume` CLI to create true virtual machines.
|
||||
|
||||
Here's what's happening behind the scenes:
|
||||
|
||||
1. The Docker container provides a consistent environment to run the Lumier interface
|
||||
2. Lumier connects to the Lume service running on your host Mac
|
||||
3. Lume uses Apple's Virtualization Framework to create a true macOS virtual machine
|
||||
4. The VM runs with hardware acceleration using your Mac's native virtualization capabilities
|
||||
<Callout type="info">
|
||||
Lume runs as a background service on port 7777. If this port is already in
|
||||
use, specify a different port with the `--port` option during installation.
|
||||
</Callout>
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Quick Start
|
||||
|
||||
Run your first macOS VM with a single Docker command:
|
||||
|
||||
```bash
|
||||
# Run a macOS VM with default settings
|
||||
docker run -it \
|
||||
-e LUME_SERVER_URL="host.docker.internal:7777" \
|
||||
-p 5900:5900 \
|
||||
ghcr.io/trycua/lumier:latest
|
||||
```
|
||||
|
||||
### Basic Configuration
|
||||
|
||||
Customize your VM with environment variables:
|
||||
|
||||
```bash
|
||||
docker run -it \
|
||||
-e LUME_SERVER_URL="host.docker.internal:7777" \
|
||||
-e VM_NAME="my-dev-vm" \
|
||||
-e VM_CPUS="8" \
|
||||
-e VM_MEMORY="16384" \
|
||||
-e VM_STORAGE="100" \
|
||||
-e VNC_PASSWORD="mysecretpassword" \
|
||||
-p 5900:5900 \
|
||||
ghcr.io/trycua/lumier:latest
|
||||
```
|
||||
|
||||
### Access Your VM
|
||||
|
||||
Once running, access your VM through:
|
||||
|
||||
1. **VNC Client**: Connect to `vnc://localhost:5900`
|
||||
2. **Web Browser**: Navigate to `http://localhost:5900` (if using noVNC)
|
||||
|
||||
## Examples
|
||||
|
||||
### Ephemeral VM (Temporary)
|
||||
|
||||
Run a VM that resets on restart - perfect for testing:
|
||||
|
||||
```bash
|
||||
# Run the container with temporary storage (using pre-built image from Docker Hub)
|
||||
docker run -it --rm \
|
||||
--name macos-vm \
|
||||
-p 8006:8006 \
|
||||
@@ -90,13 +119,14 @@ docker run -it --rm \
|
||||
trycua/lumier:latest
|
||||
```
|
||||
|
||||
After running the command above, you can access your macOS VM through a web browser (e.g., http://localhost:8006).
|
||||
<Callout type="info">
|
||||
Access your VM at `http://localhost:8006` after startup. Changes will be lost
|
||||
when the container stops.
|
||||
</Callout>
|
||||
|
||||
> **Note:** With the basic setup above, your VM will be reset when you stop the container (ephemeral mode). This means any changes you make inside the macOS VM will be lost. See the section below for how to save your VM state.
|
||||
### Persistent VM
|
||||
|
||||
## Saving Your VM State
|
||||
|
||||
To save your VM state between sessions (so your changes persist when you stop and restart the container), you'll need to set up a storage location:
|
||||
Save your VM state between sessions with persistent storage:
|
||||
|
||||
```bash
|
||||
# First, create a storage directory if it doesn't exist
|
||||
@@ -115,11 +145,9 @@ docker run -it --rm \
|
||||
trycua/lumier:latest
|
||||
```
|
||||
|
||||
This command creates a connection between a folder on your Mac (`$(pwd)/storage`) and a folder inside the Docker container (`/storage`). The `-v` flag (volume mount) and the `HOST_STORAGE_PATH` variable work together to ensure your VM data is saved on your host Mac.
|
||||
### File Sharing
|
||||
|
||||
## Sharing Files with Your VM
|
||||
|
||||
To share files between your Mac and the virtual machine, you can set up a shared folder:
|
||||
Share files between your host and VM:
|
||||
|
||||
```bash
|
||||
# Create both storage and shared folders
|
||||
@@ -140,11 +168,11 @@ docker run -it --rm \
|
||||
trycua/lumier:latest
|
||||
```
|
||||
|
||||
With this setup, any files you place in the `shared` folder on your Mac will be accessible from within the macOS VM, and vice versa.
|
||||
Files in the `shared` folder are accessible from both your Mac and the VM.
|
||||
|
||||
## Automating VM Startup with on-logon.sh
|
||||
### Automation with Startup Scripts
|
||||
|
||||
You can automatically run scripts when the VM starts up by placing an `on-logon.sh` script in the shared folder's lifecycle directory. This is useful for setting up your VM environment each time it starts.
|
||||
Automate VM setup with startup scripts:
|
||||
|
||||
```bash
|
||||
# Create the lifecycle directory in your shared folder
|
||||
@@ -169,21 +197,17 @@ EOF
|
||||
chmod +x shared/lifecycle/on-logon.sh
|
||||
```
|
||||
|
||||
The script will be automatically executed when the VM starts up. It runs in the VM context and has access to:
|
||||
The script runs automatically on VM startup with access to:
|
||||
|
||||
- The `/Users/lume` user directory (home directory in the VM)
|
||||
- The shared folder at `/Volumes/My Shared Files` inside the VM
|
||||
- Any resources available to the VM
|
||||
- Home directory: `/Users/lume`
|
||||
- Shared folder: `/Volumes/My Shared Files`
|
||||
- All VM resources
|
||||
|
||||
This feature enables automation of VM setup without modifying the base VM image.
|
||||
### Docker Compose
|
||||
|
||||
## Using Docker Compose
|
||||
|
||||
You can also use Docker Compose to run Lumier with a simple configuration file. Create a `docker-compose.yml` file with the following content:
|
||||
For easier management, use Docker Compose:
|
||||
|
||||
```yaml
|
||||
version: '3'
|
||||
|
||||
services:
|
||||
lumier:
|
||||
image: trycua/lumier:latest
|
||||
@@ -205,7 +229,7 @@ services:
|
||||
stop_grace_period: 2m
|
||||
```
|
||||
|
||||
Then run Lumier using:
|
||||
Run with Docker Compose:
|
||||
|
||||
```bash
|
||||
# First create the required directories
|
||||
@@ -221,15 +245,18 @@ docker-compose logs -f
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
## Building and Customizing Lumier
|
||||
## Advanced Topics
|
||||
|
||||
If you want to customize the Lumier container or build it from source, you can follow these steps:
|
||||
### Building from Source
|
||||
|
||||
Customize Lumier by building from source:
|
||||
|
||||
```bash
|
||||
# 1. Navigate to the Lumier directory
|
||||
cd libs/lumier
|
||||
# Clone the repository
|
||||
git clone https://github.com/trycua/cua.git
|
||||
cd cua/libs/lumier
|
||||
|
||||
# 2. Build the Docker image locally
|
||||
# Build the Docker image
|
||||
docker build -t lumier-custom:latest .
|
||||
|
||||
# 3. Run your custom build
|
||||
@@ -262,30 +289,60 @@ docker build -t yourusername/lumier:custom .
|
||||
docker push yourusername/lumier:custom
|
||||
```
|
||||
|
||||
## Configuration Options
|
||||
### Configuration Reference
|
||||
|
||||
When running Lumier, you'll need to configure a few things:
|
||||
#### Environment Variables
|
||||
|
||||
- **Port forwarding** (`-p 8006:8006`): Makes the VM's VNC interface accessible in your browser. If port 8006 is already in use, you can use a different port like `-p 8007:8006`.
|
||||
| Variable | Description | Default | Example |
|
||||
| ------------------- | --------------------------- | --------------------------- | ----------------------------------------- |
|
||||
| `LUME_SERVER_URL` | Lume service URL | `host.docker.internal:7777` | `host.docker.internal:8080` |
|
||||
| `VM_NAME` | Virtual machine name | `lumier-vm` | `my-dev-vm` |
|
||||
| `VERSION` | VM image to use | - | `ghcr.io/trycua/macos-sequoia-cua:latest` |
|
||||
| `VM_CPUS` | Number of CPU cores | `4` | `8` |
|
||||
| `VM_MEMORY` | Memory in MB | `8192` | `16384` |
|
||||
| `VM_STORAGE` | Storage size in GB | `50` | `100` |
|
||||
| `VNC_PASSWORD` | VNC access password | - | `mysecretpassword` |
|
||||
| `HOST_STORAGE_PATH` | Host path for VM storage | - | `$(pwd)/storage` |
|
||||
| `HOST_SHARED_PATH` | Host path for shared folder | - | `$(pwd)/shared` |
|
||||
|
||||
- **Environment variables** (`-e`): Configure your VM settings:
|
||||
#### Port Configuration
|
||||
|
||||
- `VM_NAME`: A name for your virtual machine
|
||||
- `VERSION`: The macOS image to use
|
||||
- `CPU_CORES`: Number of CPU cores to allocate
|
||||
- `RAM_SIZE`: Memory in MB to allocate
|
||||
- `HOST_STORAGE_PATH`: Path to save VM state (when using persistent storage)
|
||||
- `HOST_SHARED_PATH`: Path to the shared folder (optional)
|
||||
- **VNC Port**: `-p 5900:5900` for standard VNC access
|
||||
- **Web Port**: `-p 8006:8006` for browser-based access
|
||||
- Use different host ports if defaults are occupied: `-p 8007:8006`
|
||||
|
||||
- **Background service**: The `lume serve` service should be running on your host (starts automatically when you install Lume using the `install.sh` script above).
|
||||
## Resources
|
||||
|
||||
## Credits
|
||||
<Cards>
|
||||
<Card
|
||||
title="Lume Documentation"
|
||||
description="Learn more about the virtualization service powering Lumier"
|
||||
href="/home/libraries/lume"
|
||||
/>
|
||||
<Card
|
||||
title="Computer Library"
|
||||
description="Automate your VMs with the Computer library"
|
||||
href="/home/libraries/computer"
|
||||
/>
|
||||
</Cards>
|
||||
|
||||
This project was inspired by [dockur/windows](https://github.com/dockur/windows) and [dockur/macos](https://github.com/dockur/macos), which pioneered the approach of running Windows and macOS VMs in Docker containers.
|
||||
---
|
||||
|
||||
Main differences with dockur/macos:
|
||||
|
||||
- Lumier is specifically designed for macOS virtualization
|
||||
- Lumier supports Apple Silicon (M1/M2/M3/M4) while dockur/macos only supports Intel
|
||||
- Lumier uses the Apple Virtualization Framework (Vz) through the `lume` CLI to create true virtual machines, while dockur relies on KVM.
|
||||
- Image specification is different, with Lumier and Lume relying on Apple Vz spec (disk.img and nvram.bin)
|
||||
<Callout type="info">
|
||||
**Need detailed API documentation?**{' '}
|
||||
<p>
|
||||
Explore the complete API reference with detailed class documentation, and
|
||||
method signatures.
|
||||
</p>
|
||||
<a
|
||||
href="/api/lumier"
|
||||
className={cn(
|
||||
buttonVariants({
|
||||
color: 'secondary',
|
||||
}),
|
||||
'no-underline h-10'
|
||||
)}>
|
||||
View API Reference
|
||||
<ChevronRight size={18} />
|
||||
</a>
|
||||
</Callout>
|
||||
|
||||
@@ -1,88 +1,64 @@
|
||||
---
|
||||
title: MCP Server
|
||||
description: Model Context Protocol server for Computer-Use Agent integration
|
||||
pypi: cua-mcp-server
|
||||
macos: true
|
||||
linux: true
|
||||
windows: true
|
||||
---
|
||||
|
||||
<div
|
||||
align="center"
|
||||
style={{
|
||||
display: 'flex',
|
||||
gap: '10px',
|
||||
margin: '0 auto',
|
||||
width: '100%',
|
||||
justifyContent: 'center',
|
||||
}}
|
||||
>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Swift_6-F54A2A?logo=swift&logoColor=white&labelColor=F54A2A"
|
||||
alt="Swift 6"
|
||||
/>
|
||||
</a>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=F0F0F0"
|
||||
alt="macOS"
|
||||
/>
|
||||
</a>
|
||||
<a href="https://discord.com/invite/mVnXXpdE85">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Discord-%235865F2.svg?&logo=discord&logoColor=white"
|
||||
alt="Discord"
|
||||
/>
|
||||
</a>
|
||||
<a href="https://pypi.org/project/cua-computer/">
|
||||
<img
|
||||
src="https://img.shields.io/pypi/v/cua-computer?color=blue"
|
||||
alt="Python"
|
||||
/>
|
||||
</a>
|
||||
</div>
|
||||
import { buttonVariants } from 'fumadocs-ui/components/ui/button';
|
||||
import { cn } from 'fumadocs-ui/utils/cn';
|
||||
import { ChevronRight } from 'lucide-react';
|
||||
import { Step, Steps } from 'fumadocs-ui/components/steps';
|
||||
|
||||
**cua-mcp-server** is a MCP server for the Computer-Use Agent (CUA), allowing you to run CUA through Claude Desktop or other MCP clients.
|
||||
**MCP Server** enables Computer-Use Agent (CUA) integration with Claude Desktop and other Model Context Protocol (MCP) clients, providing seamless access to computer automation capabilities through a standardized interface.
|
||||
|
||||
### Get started with Agent
|
||||
## Features
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before installing the MCP server, you'll need to set up the full Computer-Use Agent capabilities as described in [Option 2 of the main README](../../README.md#option-2-full-computer-use-agent-capabilities). This includes:
|
||||
|
||||
1. Installing the Lume CLI
|
||||
2. Pulling the latest macOS CUA image
|
||||
3. Starting the Lume daemon service
|
||||
4. Installing the required Python libraries (Optional: only needed if you want to verify the agent is working before installing MCP server)
|
||||
|
||||
Make sure these steps are completed and working before proceeding with the MCP server installation.
|
||||
- **MCP Integration** - Connect CUA to Claude Desktop and other MCP-compatible clients
|
||||
- **Computer Control** - Full screen, keyboard, and mouse automation capabilities
|
||||
- **Tool System** - Execute commands, take screenshots, and interact with applications
|
||||
- **Easy Setup** - Simple configuration with Claude Desktop or any MCP client
|
||||
|
||||
## Installation
|
||||
|
||||
Install the package from PyPI:
|
||||
### Prerequisites
|
||||
|
||||
Before installing the MCP server, ensure you have:
|
||||
|
||||
1. **Lume CLI** installed and configured
|
||||
2. **macOS CUA image** pulled and ready
|
||||
3. **Python 3.10+** installed on your system
|
||||
|
||||
### Install via pip
|
||||
|
||||
```bash
|
||||
pip install cua-mcp-server
|
||||
```
|
||||
|
||||
This will install:
|
||||
### Quick Setup Script
|
||||
|
||||
- The MCP server
|
||||
- CUA agent and computer dependencies
|
||||
- An executable `cua-mcp-server` script in your PATH
|
||||
|
||||
## Easy Setup Script
|
||||
|
||||
If you want to simplify installation, you can use this one-liner to download and run the installation script:
|
||||
For automated installation, use our setup script:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/mcp-server/scripts/install_mcp_server.sh | bash
|
||||
```
|
||||
|
||||
This script will:
|
||||
This script:
|
||||
|
||||
- Create the ~/.cua directory if it doesn't exist
|
||||
- Generate a startup script at ~/.cua/start_mcp_server.sh
|
||||
- Make the script executable
|
||||
- The startup script automatically manages Python virtual environments and installs/updates the cua-mcp-server package
|
||||
- Creates the `~/.cua` directory
|
||||
- Generates a startup script at `~/.cua/start_mcp_server.sh`
|
||||
- Manages Python virtual environments automatically
|
||||
- Installs and updates the cua-mcp-server package
|
||||
|
||||
You can then use the script in your MCP configuration like this:
|
||||
## Getting Started
|
||||
|
||||
<Steps>
|
||||
<Step>
|
||||
### Configure Claude Desktop
|
||||
|
||||
Add the MCP server to your Claude Desktop configuration:
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -93,81 +69,35 @@ You can then use the script in your MCP configuration like this:
|
||||
"env": {
|
||||
"CUA_AGENT_LOOP": "OMNI",
|
||||
"CUA_MODEL_PROVIDER": "ANTHROPIC",
|
||||
"CUA_MODEL_NAME": "claude-3-7-sonnet-20250219",
|
||||
"CUA_PROVIDER_API_KEY": "your-api-key"
|
||||
"CUA_PROVIDER_API_KEY": "your-anthropic-api-key"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Development Guide
|
||||
<Callout type="tip">
|
||||
The configuration file is typically located at
|
||||
`~/.config/claude-desktop/claude_desktop_config.json`
|
||||
</Callout>
|
||||
</Step>
|
||||
|
||||
If you want to develop with the cua-mcp-server directly without installation, you can use this configuration:
|
||||
<Step>
|
||||
### Start Using CUA
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"cua-agent": {
|
||||
"command": "/bin/bash",
|
||||
"args": ["~/cua/libs/mcp-server/scripts/start_mcp_server.sh"],
|
||||
"env": {
|
||||
"CUA_AGENT_LOOP": "UITARS",
|
||||
"CUA_MODEL_PROVIDER": "OAICOMPAT",
|
||||
"CUA_MODEL_NAME": "ByteDance-Seed/UI-TARS-1.5-7B",
|
||||
"CUA_PROVIDER_BASE_URL": "https://****************.us-east-1.aws.endpoints.huggingface.cloud/v1",
|
||||
"CUA_PROVIDER_API_KEY": "your-api-key"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Once configured, you can ask Claude to perform computer tasks:
|
||||
|
||||
```text
|
||||
"Take a screenshot of the current screen"
|
||||
"Open Safari and navigate to google.com"
|
||||
"Click on the search bar and type 'weather'"
|
||||
```
|
||||
</Step>
|
||||
|
||||
This configuration:
|
||||
<Step>
|
||||
### Configure Environment Variables
|
||||
|
||||
- Uses the start_mcp_server.sh script which automatically sets up the Python path and runs the server module
|
||||
- Works with Claude Desktop, Cursor, or any other MCP client
|
||||
- Automatically uses your development code without requiring installation
|
||||
|
||||
Just add this to your MCP client's configuration and it will use your local development version of the server.
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
If you get a `/bin/bash: ~/cua/libs/mcp-server/scripts/start_mcp_server.sh: No such file or directory` error, try changing the path to the script to be absolute instead of relative.
|
||||
|
||||
To see the logs:
|
||||
|
||||
```
|
||||
tail -n 20 -f ~/Library/Logs/Claude/mcp*.log
|
||||
```
|
||||
|
||||
## Claude Desktop Integration
|
||||
|
||||
To use with Claude Desktop, add an entry to your Claude Desktop configuration (`claude_desktop_config.json`, typically found in `~/.config/claude-desktop/`):
|
||||
|
||||
For more information on MCP with Claude Desktop, see the [official MCP User Guide](https://modelcontextprotocol.io/quickstart/user).
|
||||
|
||||
## Cursor Integration
|
||||
|
||||
To use with Cursor, add an MCP configuration file in one of these locations:
|
||||
|
||||
- **Project-specific**: Create `.cursor/mcp.json` in your project directory
|
||||
- **Global**: Create `~/.cursor/mcp.json` in your home directory
|
||||
|
||||
After configuration, you can simply tell Cursor's Agent to perform computer tasks by explicitly mentioning the CUA agent, such as "Use the computer control tools to open Safari."
|
||||
|
||||
For more information on MCP with Cursor, see the [official Cursor MCP documentation](https://docs.cursor.com/context/model-context-protocol).
|
||||
|
||||
### First-time Usage Notes
|
||||
|
||||
**API Keys**: Ensure you have valid API keys:
|
||||
|
||||
- Add your Anthropic API key, or other model provider API key in the Claude Desktop config (as shown above)
|
||||
- Or set it as an environment variable in your shell profile
|
||||
|
||||
## Configuration
|
||||
|
||||
The server is configured using environment variables (can be set in the Claude Desktop config):
|
||||
Customize the MCP server behavior with these environment variables:
|
||||
|
||||
| Variable | Description | Default |
|
||||
| ----------------------- | ----------------------------------------------------- | ----------------------- |
|
||||
@@ -176,21 +106,95 @@ The server is configured using environment variables (can be set in the Claude D
|
||||
| `CUA_MODEL_NAME` | Model name to use | None (provider default) |
|
||||
| `CUA_PROVIDER_BASE_URL` | Base URL for provider API | None |
|
||||
| `CUA_MAX_IMAGES` | Maximum number of images to keep in context | 3 |
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
## Available Tools
|
||||
## Examples
|
||||
|
||||
The MCP server exposes the following tools to Claude:
|
||||
### Basic Computer Control
|
||||
|
||||
1. `run_cua_task` - Run a single Computer-Use Agent task with the given instruction
|
||||
2. `run_multi_cua_tasks` - Run multiple tasks in sequence
|
||||
```python
|
||||
# Example prompt to Claude
|
||||
"Take a screenshot and save it to the desktop"
|
||||
|
||||
## Usage
|
||||
# Claude will use the run_cua_task tool:
|
||||
# {
|
||||
# "instruction": "Take a screenshot and save it to the desktop",
|
||||
# "display": "1920x1080"
|
||||
# }
|
||||
```
|
||||
|
||||
Once configured, you can simply ask Claude to perform computer tasks:
|
||||
### Multi-Step Automation
|
||||
|
||||
- "Open Chrome and go to github.com"
|
||||
- "Create a folder called 'Projects' on my desktop"
|
||||
- "Find all PDFs in my Downloads folder"
|
||||
- "Take a screenshot and highlight the error message"
|
||||
```python
|
||||
# Example prompt to Claude
|
||||
"Open a new browser window, navigate to github.com, and search for 'cua'"
|
||||
|
||||
Claude will automatically use your CUA agent to perform these tasks.
|
||||
# Claude will use the run_multi_cua_tasks tool with multiple instructions
|
||||
```
|
||||
|
||||
### Integrating with Cursor
|
||||
|
||||
Create a configuration file at `~/.cursor/mcp.json` or `.cursor/mcp.json` in your project:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"cua-agent": {
|
||||
"command": "/bin/bash",
|
||||
"args": ["~/.cua/start_mcp_server.sh"],
|
||||
"env": {
|
||||
"CUA_AGENT_LOOP": "OMNI",
|
||||
"CUA_MODEL_PROVIDER": "ANTHROPIC",
|
||||
"CUA_PROVIDER_API_KEY": "your-api-key"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Development Mode
|
||||
|
||||
For local development without installation:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"cua-agent": {
|
||||
"command": "/bin/bash",
|
||||
"args": ["~/cua/libs/mcp-server/scripts/start_mcp_server.sh"],
|
||||
"env": {
|
||||
"CUA_AGENT_LOOP": "OMNI",
|
||||
"CUA_MODEL_PROVIDER": "ANTHROPIC",
|
||||
"CUA_PROVIDER_API_KEY": "your-api-key"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
View MCP server logs:
|
||||
|
||||
```bash
|
||||
tail -n 20 -f ~/Library/Logs/Claude/mcp*.log
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
<Callout type="info">
|
||||
**Need detailed API documentation?** Explore the complete API reference with
|
||||
detailed tool descriptions, parameters, and usage examples.
|
||||
<a
|
||||
href="/api/mcp-server"
|
||||
className={cn(
|
||||
buttonVariants({
|
||||
color: 'secondary',
|
||||
}),
|
||||
'no-underline h-10'
|
||||
)}>
|
||||
View API Reference
|
||||
<ChevronRight size={18} />
|
||||
</a>
|
||||
</Callout>
|
||||
|
||||
@@ -1,217 +1,294 @@
|
||||
---
|
||||
title: Set-of-Mark
|
||||
macos: true
|
||||
pypi: cua-computer
|
||||
---
|
||||
|
||||
<div
|
||||
align="center"
|
||||
style={{
|
||||
display: 'flex',
|
||||
gap: '10px',
|
||||
margin: '0 auto',
|
||||
width: '100%',
|
||||
justifyContent: 'center',
|
||||
}}
|
||||
>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Python-333333?logo=python&logoColor=white&labelColor=blue"
|
||||
alt="Python"
|
||||
/>
|
||||
</a>
|
||||
<a href="#">
|
||||
<img
|
||||
src="https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=F0F0F0"
|
||||
alt="macOS"
|
||||
/>
|
||||
</a>
|
||||
<a href="https://discord.com/invite/mVnXXpdE85">
|
||||
<img
|
||||
src="https://img.shields.io/badge/Discord-%235865F2.svg?&logo=discord&logoColor=white"
|
||||
alt="Discord"
|
||||
/>
|
||||
</a>
|
||||
<a href="https://pypi.org/project/cua-computer/">
|
||||
<img
|
||||
src="https://img.shields.io/pypi/v/cua-computer?color=blue"
|
||||
alt="PyPI"
|
||||
/>
|
||||
</a>
|
||||
</div>
|
||||
import { buttonVariants } from 'fumadocs-ui/components/ui/button';
|
||||
import { cn } from 'fumadocs-ui/utils/cn';
|
||||
import { ChevronRight } from 'lucide-react';
|
||||
|
||||
**Som** (Set-of-Mark) is a visual grounding component for the Computer-Use Agent (CUA) framework powering Cua, for detecting and analyzing UI elements in screenshots. Optimized for macOS Silicon with Metal Performance Shaders (MPS), it combines YOLO-based icon detection with EasyOCR text recognition to provide comprehensive UI element analysis.
|
||||
## Overview
|
||||
|
||||
## Features
|
||||
**Set-of-Mark (Som)** is a high-performance visual grounding library for detecting and analyzing UI elements in screenshots. Built for the Computer-Use Agent (CUA) framework, it combines state-of-the-art computer vision models to identify icons, buttons, and text in user interfaces.
|
||||
|
||||
- Optimized for Apple Silicon with MPS acceleration
|
||||
- Icon detection using YOLO with multi-scale processing
|
||||
- Text recognition using EasyOCR (GPU-accelerated)
|
||||
- Automatic hardware detection (MPS → CUDA → CPU)
|
||||
- Smart detection parameters tuned for UI elements
|
||||
- Detailed visualization with numbered annotations
|
||||
- Performance benchmarking tools
|
||||
<Callout type="info">
|
||||
Som is optimized for **Apple Silicon** with Metal Performance Shaders (MPS)
|
||||
acceleration, achieving sub-second detection times while maintaining high
|
||||
accuracy.
|
||||
</Callout>
|
||||
|
||||
## System Requirements
|
||||
### Key Features
|
||||
|
||||
- **Recommended**: macOS with Apple Silicon
|
||||
- Uses Metal Performance Shaders (MPS)
|
||||
- Multi-scale detection enabled
|
||||
- ~0.4s average detection time
|
||||
- **Supported**: Any Python 3.11+ environment
|
||||
- Falls back to CPU if no GPU available
|
||||
- Single-scale detection on CPU
|
||||
- ~1.3s average detection time
|
||||
- **Hardware Acceleration** - Automatic detection of MPS, CUDA, or CPU
|
||||
- **Multi-Model Architecture** - YOLO for icons + EasyOCR for text
|
||||
- **Optimized Performance** - Sub-second detection on Apple Silicon
|
||||
- **Flexible Configuration** - Tunable thresholds for different use cases
|
||||
- **Rich Output Format** - Structured data with confidence scores
|
||||
- **Visual Debugging** - Annotated screenshots with numbered elements
|
||||
|
||||
## Installation
|
||||
|
||||
<Callout type="warning">
|
||||
Som requires Python 3.11 or higher. For best performance, use macOS with Apple
|
||||
Silicon.
|
||||
</Callout>
|
||||
|
||||
### Install from PyPI
|
||||
|
||||
```bash
|
||||
pip install cua-som
|
||||
```
|
||||
|
||||
### Install from Source
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone https://github.com/cua/som.git
|
||||
cd som
|
||||
|
||||
# Using PDM (recommended)
|
||||
pdm install
|
||||
|
||||
# Using pip
|
||||
# Or using pip
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
### System Requirements
|
||||
|
||||
| Platform | Hardware | Detection Time |
|
||||
| ------------- | ------------------------ | -------------- |
|
||||
| macOS | Apple Silicon (M1/M2/M3) | ~0.4s |
|
||||
| Linux/Windows | NVIDIA GPU | ~0.6s |
|
||||
| Any | CPU only | ~1.3s |
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Basic Usage
|
||||
|
||||
Here's a simple example to detect UI elements in a screenshot:
|
||||
|
||||
```python
|
||||
from som import OmniParser
|
||||
from PIL import Image
|
||||
|
||||
# Initialize parser
|
||||
# Initialize the parser
|
||||
parser = OmniParser()
|
||||
|
||||
# Process an image
|
||||
# Load and process an image
|
||||
image = Image.open("screenshot.png")
|
||||
result = parser.parse(
|
||||
image,
|
||||
box_threshold=0.3, # Confidence threshold
|
||||
iou_threshold=0.1, # Overlap threshold
|
||||
use_ocr=True # Enable text detection
|
||||
)
|
||||
result = parser.parse(image)
|
||||
|
||||
# Access results
|
||||
# Print detected elements
|
||||
for elem in result.elements:
|
||||
if elem.type == "icon":
|
||||
print(f"Icon: confidence={elem.confidence:.3f}, bbox={elem.bbox.coordinates}")
|
||||
else: # text
|
||||
print(f"Text: '{elem.content}', confidence={elem.confidence:.3f}")
|
||||
print(f"{elem.type}: {elem.content or 'icon'} at {elem.bbox.coordinates}")
|
||||
```
|
||||
|
||||
## Configuration
|
||||
### Advanced Configuration
|
||||
|
||||
Customize detection parameters for your specific use case:
|
||||
|
||||
```python
|
||||
result = parser.parse(
|
||||
image,
|
||||
box_threshold=0.3, # Confidence threshold (0.0-1.0)
|
||||
iou_threshold=0.1, # Overlap threshold (0.0-1.0)
|
||||
use_ocr=True, # Enable text detection
|
||||
ocr_engine="easyocr" # OCR engine choice
|
||||
)
|
||||
```
|
||||
|
||||
### Working with Results
|
||||
|
||||
```python
|
||||
# Filter by element type
|
||||
icons = [e for e in result.elements if e.type == "icon"]
|
||||
texts = [e for e in result.elements if e.type == "text"]
|
||||
|
||||
# Get high-confidence detections
|
||||
high_conf = [e for e in result.elements if e.confidence > 0.8]
|
||||
|
||||
# Access bounding boxes
|
||||
for elem in result.elements:
|
||||
x, y, w, h = elem.bbox.coordinates
|
||||
print(f"Element at ({x}, {y}) with size {w}x{h}")
|
||||
```
|
||||
|
||||
<Callout>
|
||||
💡 **Tip**: Check out our [interactive examples](#examples) below to see Som
|
||||
in action with real UI screenshots.
|
||||
</Callout>
|
||||
|
||||
## Configuration Guide
|
||||
|
||||
### Detection Parameters
|
||||
|
||||
#### Box Threshold (0.3)
|
||||
<Cards>
|
||||
<Card
|
||||
title="Box Threshold"
|
||||
description="Controls detection confidence (default: 0.3)">
|
||||
- **Higher values (0.4-0.5)**: More precise, fewer false positives - **Lower
|
||||
values (0.1-0.2)**: More detections, may include noise - **Recommended**:
|
||||
0.3 for balanced performance
|
||||
</Card>
|
||||
|
||||
Controls the confidence threshold for accepting detections:
|
||||
<Card
|
||||
title="IOU Threshold"
|
||||
description="Controls overlap handling (default: 0.1)">
|
||||
- **Lower values (0.05-0.1)**: Aggressive merging of overlaps - **Higher
|
||||
values (0.3-0.5)**: Keeps more overlapping boxes - **Recommended**: 0.1 for
|
||||
dense UIs
|
||||
</Card>
|
||||
</Cards>
|
||||
|
||||
```
|
||||
High Threshold (0.3): Low Threshold (0.01):
|
||||
+----------------+ +----------------+
|
||||
| | | +--------+ |
|
||||
| Confident | | |Unsure?| |
|
||||
| Detection | | +--------+ |
|
||||
| (✓ Accept) | | (? Reject) |
|
||||
| | | |
|
||||
+----------------+ +----------------+
|
||||
conf = 0.85 conf = 0.02
|
||||
```
|
||||
### OCR Settings
|
||||
|
||||
- Higher values (0.3) yield more precise but fewer detections
|
||||
- Lower values (0.01) catch more potential icons but increase false positives
|
||||
- Default is 0.3 for optimal precision/recall balance
|
||||
| Setting | Default | Description |
|
||||
| -------------- | --------- | ------------------------ |
|
||||
| Engine | `easyocr` | OCR engine to use |
|
||||
| Languages | `['en']` | Supported languages |
|
||||
| GPU | `auto` | Enable GPU acceleration |
|
||||
| Timeout | `5s` | Maximum processing time |
|
||||
| Min Confidence | `0.5` | Text detection threshold |
|
||||
|
||||
#### IOU Threshold (0.1)
|
||||
|
||||
Controls how overlapping detections are merged:
|
||||
|
||||
```
|
||||
IOU = Intersection Area / Union Area
|
||||
|
||||
Low Overlap (Keep Both): High Overlap (Merge):
|
||||
+----------+ +----------+
|
||||
| Box1 | | Box1 |
|
||||
| | vs. |+-----+ |
|
||||
+----------+ ||Box2 | |
|
||||
+----------+ |+-----+ |
|
||||
| Box2 | +----------+
|
||||
| |
|
||||
+----------+
|
||||
IOU ≈ 0.05 (Keep Both) IOU ≈ 0.7 (Merge)
|
||||
```
|
||||
|
||||
- Lower values (0.1) more aggressively remove overlapping boxes
|
||||
- Higher values (0.5) allow more overlapping detections
|
||||
- Default is 0.1 to handle densely packed UI elements
|
||||
|
||||
### OCR Configuration
|
||||
|
||||
- **Engine**: EasyOCR
|
||||
|
||||
- Primary choice for all platforms
|
||||
- Fast initialization and processing
|
||||
- Built-in English language support
|
||||
- GPU acceleration when available
|
||||
|
||||
- **Settings**:
|
||||
- Timeout: 5 seconds
|
||||
- Confidence threshold: 0.5
|
||||
- Paragraph mode: Disabled
|
||||
- Language: English only
|
||||
|
||||
## Performance
|
||||
## Performance Benchmarks
|
||||
|
||||
### Hardware Acceleration
|
||||
|
||||
#### MPS (Metal Performance Shaders)
|
||||
<Mermaid
|
||||
chart="
|
||||
graph LR
|
||||
A[Input Image] --> B{Hardware Detection}
|
||||
B -->|Apple Silicon| C[MPS Backend]
|
||||
B -->|NVIDIA GPU| D[CUDA Backend]
|
||||
B -->|No GPU| E[CPU Backend]
|
||||
C --> F[Multi-scale\n~0.4s]
|
||||
D --> G[Multi-scale\n~0.6s]
|
||||
E --> H[Single-scale\n~1.3s]
|
||||
"
|
||||
/>
|
||||
|
||||
- Multi-scale detection (640px, 1280px, 1920px)
|
||||
- Test-time augmentation enabled
|
||||
- Half-precision (FP16)
|
||||
- Average detection time: ~0.4s
|
||||
- Best for production use when available
|
||||
### Optimization Details
|
||||
|
||||
#### CPU
|
||||
<Cards>
|
||||
<Card title="MPS (Apple Silicon)" description="Best performance on macOS">
|
||||
- Multi-scale detection (640px, 1280px, 1920px) - Test-time augmentation -
|
||||
Half-precision (FP16) - ~0.4s average time
|
||||
</Card>
|
||||
|
||||
- Single-scale detection (1280px)
|
||||
- Full-precision (FP32)
|
||||
- Average detection time: ~1.3s
|
||||
- Reliable fallback option
|
||||
{' '}
|
||||
|
||||
### Example Output Structure
|
||||
<Card title="CUDA (NVIDIA)" description="High performance on Linux/Windows">
|
||||
- Multi-scale detection - Mixed precision - ~0.6s average time
|
||||
</Card>
|
||||
|
||||
```
|
||||
examples/output/
|
||||
├── {timestamp}_no_ocr/
|
||||
│ ├── annotated_images/
|
||||
│ │ └── screenshot_analyzed.png
|
||||
│ ├── screen_details.txt
|
||||
│ └── summary.json
|
||||
└── {timestamp}_ocr/
|
||||
├── annotated_images/
|
||||
│ └── screenshot_analyzed.png
|
||||
├── screen_details.txt
|
||||
└── summary.json
|
||||
<Card title="CPU Fallback" description="Universal compatibility">
|
||||
- Single-scale detection (1280px) - Full precision (FP32) - ~1.3s average
|
||||
time
|
||||
</Card>
|
||||
</Cards>
|
||||
|
||||
## Examples
|
||||
|
||||
### Example 1: Screenshot Analysis
|
||||
|
||||
Analyze a full application screenshot with visualization:
|
||||
|
||||
```python
|
||||
from som import OmniParser
|
||||
from PIL import Image
|
||||
import json
|
||||
|
||||
# Initialize parser
|
||||
parser = OmniParser()
|
||||
|
||||
# Load screenshot
|
||||
image = Image.open("app_screenshot.png")
|
||||
|
||||
# Parse with visualization
|
||||
result = parser.parse(image, use_ocr=True)
|
||||
|
||||
# Save annotated image
|
||||
result.visualized_image.save("annotated_screenshot.png")
|
||||
|
||||
# Export results as JSON
|
||||
with open("ui_elements.json", "w") as f:
|
||||
json.dump(result.to_dict(), f, indent=2)
|
||||
```
|
||||
|
||||
## Development
|
||||
### Example 2: UI Automation Helper
|
||||
|
||||
### Test Data
|
||||
Find and interact with specific UI elements:
|
||||
|
||||
- Place test screenshots in `examples/test_data/`
|
||||
- Not tracked in git to keep repository size manageable
|
||||
- Default test image: `test_screen.png` (1920x1080)
|
||||
```python
|
||||
def find_button(parser, image, button_text):
|
||||
"""Find a button by its text content"""
|
||||
result = parser.parse(image, use_ocr=True)
|
||||
|
||||
### Running Tests
|
||||
for elem in result.elements:
|
||||
if elem.type == "text" and button_text.lower() in elem.content.lower():
|
||||
return elem.bbox.coordinates
|
||||
|
||||
```bash
|
||||
# Run benchmark with no OCR
|
||||
python examples/omniparser_examples.py examples/test_data/test_screen.png --runs 5 --ocr none
|
||||
return None
|
||||
|
||||
# Run benchmark with OCR
|
||||
python examples/omniparser_examples.py examples/test_data/test_screen.png --runs 5 --ocr easyocr
|
||||
# Usage
|
||||
button_coords = find_button(parser, screenshot, "Submit")
|
||||
if button_coords:
|
||||
x, y, w, h = button_coords
|
||||
click_x = x + w // 2
|
||||
click_y = y + h // 2
|
||||
print(f"Click at ({click_x}, {click_y})")
|
||||
```
|
||||
|
||||
## License
|
||||
### Example 3: Batch Processing
|
||||
|
||||
MIT License - See LICENSE file for details.
|
||||
Process multiple screenshots with performance tracking:
|
||||
|
||||
```python
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
def batch_process(image_dir, output_dir):
|
||||
parser = OmniParser()
|
||||
image_files = Path(image_dir).glob("*.png")
|
||||
|
||||
results = []
|
||||
for img_path in image_files:
|
||||
start = time.time()
|
||||
|
||||
image = Image.open(img_path)
|
||||
result = parser.parse(image)
|
||||
|
||||
# Save annotated version
|
||||
out_path = Path(output_dir) / f"{img_path.stem}_annotated.png"
|
||||
result.visualized_image.save(out_path)
|
||||
|
||||
elapsed = time.time() - start
|
||||
results.append({
|
||||
"file": img_path.name,
|
||||
"elements_found": len(result.elements),
|
||||
"processing_time": elapsed
|
||||
})
|
||||
|
||||
print(f"Processed {img_path.name}: {len(result.elements)} elements in {elapsed:.2f}s")
|
||||
|
||||
return results
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
<Callout type="info">
|
||||
**Need the full API documentation?** Explore detailed class references, method
|
||||
signatures, and advanced configuration options.
|
||||
<a
|
||||
href="/api/som"
|
||||
className={cn(
|
||||
buttonVariants({
|
||||
color: 'secondary',
|
||||
}),
|
||||
'no-underline h-10'
|
||||
)}>
|
||||
View API Reference
|
||||
<ChevronRight size={18} />
|
||||
</a>
|
||||
</Callout>
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
"fumadocs-mdx": "11.6.7",
|
||||
"fumadocs-ui": "15.5.1",
|
||||
"lucide-react": "^0.525.0",
|
||||
"mermaid": "^11.8.1",
|
||||
"next": "15.3.3",
|
||||
"next-themes": "^0.4.6",
|
||||
"react": "^19.1.0",
|
||||
"react-dom": "^19.1.0",
|
||||
"tailwind-merge": "^3.3.1",
|
||||
|
||||
992
docs/pnpm-lock.yaml
generated
992
docs/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -57,11 +57,12 @@ export default async function Page(props: {
|
||||
return (
|
||||
<div className="flex flex-col gap-2 mx-4">
|
||||
{hasAnyPlatform && (
|
||||
<div className="flex flex-row gap-2 items-center mx-2 h-5 dark:text-neutral-400">
|
||||
<div className="flex flex-row gap-2 items-center mx-auto dark:text-neutral-400">
|
||||
{windows && (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="currentColor"
|
||||
className="h-5"
|
||||
viewBox="0 0 448 512">
|
||||
<title>Windows</title>
|
||||
<path d="M0 93.7l183.6-25.3v177.4H0V93.7zm0 324.6l183.6 25.3V268.4H0v149.9zm203.8 28L448 480V268.4H203.8v177.9zm0-380.6v180.1H448V32L203.8 65.7z" />
|
||||
@@ -71,6 +72,7 @@ export default async function Page(props: {
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="currentColor"
|
||||
className="h-5"
|
||||
viewBox="0 0 384 512">
|
||||
<title>macOS</title>
|
||||
<path d="M318.7 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7C63.3 141.2 4 184.8 4 273.5q0 39.3 14.4 81.2c12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zm-56.6-164.2c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z" />
|
||||
@@ -80,6 +82,7 @@ export default async function Page(props: {
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="currentColor"
|
||||
className="h-5"
|
||||
viewBox="0 0 448 512">
|
||||
<title>Linux</title>
|
||||
<path d="M220.8 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zm-21.9 1.7c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72C311.1 85.4 315.7 .1 234.8 0 132.4-.2 158 103.4 156.9 135.2c-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7v.1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.7 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-10 14.1-10.5 4.7-4.4 13.5-14.2 27.9-14.2 7.1 0 15.6 2.3 25.9 8.9 6.3 4.1 11.3 4.4 22.6 9.3 8.4 3.5 13.7 9.7 10.5 18.2-2.6 7.1-11 14.4-22.7 18.1-11.1 3.6-19.8 16-38.2 14.9-3.9-.2-7-1-9.6-2.1-8-3.5-12.2-10.4-20-15-8.6-4.8-13.2-10.4-14.7-15.3-1.4-4.9 0-9 4.2-12.3zm3.3 334c-2.7 35.1-43.9 34.4-75.3 18-29.9-15.8-68.6-6.5-76.5-21.9-2.4-4.7-2.4-12.7 2.6-26.4v-.2c2.4-7.6 .6-16-.6-23.9-1.2-7.8-1.8-15 .9-20 3.5-6.7 8.5-9.1 14.8-11.3 10.3-3.7 11.8-3.4 19.6-9.9 5.5-5.7 9.5-12.9 14.3-18 5.1-5.5 10-8.1 17.7-6.9 8.1 1.2 15.1 6.8 21.9 16l19.6 35.6c9.5 19.9 43.1 48.4 41 68.9zm-1.4-25.9c-4.1-6.6-9.6-13.6-14.4-19.6 7.1 0 14.2-2.2 16.7-8.9 2.3-6.2 0-14.9-7.4-24.9-13.5-18.2-38.3-32.5-38.3-32.5-13.5-8.4-21.1-18.7-24.6-29.9s-3-23.3-.3-35.2c5.2-22.9 18.6-45.2 27.2-59.2 2.3-1.7 .8 3.2-8.7 20.8-8.5 16.1-24.4 53.3-2.6 82.4 .6-20.7 5.5-41.8 13.8-61.5 12-27.4 37.3-74.9 39.3-112.7 1.1 .8 4.6 3.2 6.2 4.1 4.6 2.7 8.1 6.7 12.6 10.3 12.4 10 28.5 9.2 42.4 1.2 6.2-3.5 11.2-7.5 15.9-9 9.9-3.1 17.8-8.6 22.3-15 7.7 30.4 25.7 74.3 37.2 95.7 6.1 11.4 18.3 35.5 23.6 64.6 3.3-.1 7 .4 10.9 1.4 13.8-35.7-11.7-74.2-23.3-84.9-4.7-4.6-4.9-6.6-2.6-6.5 12.6 11.2 29.2 33.7 35.2 59 2.8 11.6 3.3 23.7 .4 35.7 16.4 6.8 35.9 17.9 30.7 34.8-2.2-.1-3.2 0-4.2 0 3.2-10.1-3.9-17.6-22.8-26.1-19.6-8.6-36-8.6-38.3 12.5-12.1 4.2-18.3 14.7-21.4 27.3-2.8 11.2-3.6 24.7-4.4 39.9-.5 7.7-3.6 18-6.8 29-32.1 22.9-76.7 32.9-114.3 7.2zm257.4-11.5c-.9 16.8-41.2 19.9-63.2 46.5-13.2 15.7-29.4 24.4-43.6 25.5s-26.5-4.8-33.7-19.3c-4.7-11.1-2.4-23.1 1.1-36.3 3.7-14.2 9.2-28.8 9.9-40.6 .8-15.2 1.7-28.5 4.2-38.7 2.6-10.3 6.6-17.2 13.7-21.1 .3-.2 .7-.3 1-.5 .8 13.2 7.3 26.6 18.8 29.5 12.6 3.3 30.7-7.5 38.4-16.3 9-.3 15.7-.9 22.6 5.1 9.9 8.5 7.1 30.3 17.1 41.6 10.6 11.6 14 19.5 13.7 24.6zM173.3 148.7c2 1.9 4.7 4.5 8 7.1 6.6 5.2 15.8 10.6 27.3 10.6 11.6 0 22.5-5.9 31.8-10.8 4.9-2.6 10.9-7 14.8-10.4s5.9-6.3 3.1-6.6-2.6 2.6-6 5.1c-4.4 3.2-9.7 7.4-13.9 9.8-7.4 4.2-19.5 10.2-29.9 10.2s-18.7-4.8-24.9-9.7c-3.1-2.5-5.7-5-7.7-6.9-1.5-1.4-1.9-4.6-4.3-4.9-1.4-.1-1.8 3.7 1.7 6.5z" />
|
||||
|
||||
47
docs/src/components/mermaid.tsx
Normal file
47
docs/src/components/mermaid.tsx
Normal file
@@ -0,0 +1,47 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useId, useRef, useState } from 'react';
|
||||
import { useTheme } from 'next-themes';
|
||||
|
||||
export function Mermaid({ chart }: { chart: string }) {
|
||||
const id = useId();
|
||||
const [svg, setSvg] = useState('');
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const currentChartRef = useRef<string>(null);
|
||||
const { resolvedTheme } = useTheme();
|
||||
|
||||
useEffect(() => {
|
||||
if (currentChartRef.current === chart || !containerRef.current) return;
|
||||
const container = containerRef.current;
|
||||
currentChartRef.current = chart;
|
||||
|
||||
async function renderChart() {
|
||||
const { default: mermaid } = await import('mermaid');
|
||||
|
||||
try {
|
||||
// configure mermaid
|
||||
mermaid.initialize({
|
||||
startOnLoad: false,
|
||||
securityLevel: 'loose',
|
||||
fontFamily: 'inherit',
|
||||
themeCSS: 'margin: 1.5rem auto 0;',
|
||||
theme: resolvedTheme === 'dark' ? 'dark' : 'default',
|
||||
});
|
||||
|
||||
const { svg, bindFunctions } = await mermaid.render(
|
||||
id,
|
||||
chart.replaceAll('\\n', '\n'),
|
||||
);
|
||||
|
||||
bindFunctions?.(container);
|
||||
setSvg(svg);
|
||||
} catch (error) {
|
||||
console.error('Error while rendering mermaid', error);
|
||||
}
|
||||
}
|
||||
|
||||
void renderChart();
|
||||
}, [chart, id, resolvedTheme]);
|
||||
|
||||
return <div ref={containerRef} dangerouslySetInnerHTML={{ __html: svg }} />;
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
import defaultMdxComponents from 'fumadocs-ui/mdx';
|
||||
import type { MDXComponents } from 'mdx/types';
|
||||
import { Mermaid } from './components/mermaid';
|
||||
|
||||
// use this function to get MDX components, you will need it for rendering MDX
|
||||
export function getMDXComponents(components?: MDXComponents): MDXComponents {
|
||||
return {
|
||||
...defaultMdxComponents,
|
||||
Mermaid,
|
||||
...components,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user