mirror of
https://github.com/trycua/lume.git
synced 2026-04-21 19:18:56 -05:00
feat(lume, computer-server): VNC backend rewrite and VirtioFS port discovery (#1205)
* feat(lume, computer-server): VNC backend rewrite and VirtioFS port discovery Rewrite the VNC backend to use Twisted's global reactor directly instead of vncdotool's api.connect(), which deadlocks inside uvicorn/asyncio on Python 3.13. Each VNC operation now creates a fresh connection via reactor.callFromThread() with threading.Event synchronization. Add VirtioFS-based VNC port discovery: lume creates a shared "lume-config" directory, writes vnc.env (port + password) after VNC starts, and the guest reads it on boot — eliminating hardcoded VNC port/password defaults. Key changes: - vnc.py: global reactor pattern, @defer.inlineCallbacks closures, manual drag (replaces mouseDrag/doPoll), client.screen fix - VM.swift: VirtioFS lume-config share, VNC URL parsing fix (URLComponents with vnc:// → http:// replacement) - cli.py: only override CUA_VNC_PORT from CLI when explicitly provided - setup-cua.sh: renamed from setup-cua-computer.sh, added "already mounted" check for lume-config VirtioFS share Tested E2E: 19/19 operations passing (screenshot, click, type, scroll, drag, hotkey, cursor position, screen size). * fix: CI failures — isort, MockVM signature, and doc sync - Fix isort ordering in vnc.py (twisted.internet imports) - Add vncPassword parameter to MockVM.run() to match VM superclass - Regenerate lume CLI and HTTP API reference docs * fix: close VNC transport after each operation and clear stale vnc.env - vnc.py: close client.transport in finally block of _with_client() to prevent TCP connection leaks accumulating over multiple operations - VM.swift: remove old vnc.env before starting VM so the guest can't read stale port/password from a previous run * style: apply black formatting to vnc.py
This commit is contained in:
Francesco Bonacci
committed by
GitHub
GitHub
parent
412e9e40e9
commit
f73a68be2a
@@ -7,16 +7,16 @@ description: Command Line Interface reference for Lume
|
||||
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
|
||||
Generated by: npx tsx scripts/docs-generators/lume.ts
|
||||
Source: libs/lume/src/Commands/*.swift
|
||||
Version: 0.2.86
|
||||
Version: 0.3.0
|
||||
*/}
|
||||
|
||||
import { Callout } from 'fumadocs-ui/components/callout';
|
||||
import { VersionHeader } from '@/components/version-selector';
|
||||
|
||||
<VersionHeader
|
||||
versions={[{"version":"0.2","href":"/lume/reference/cli-reference","isCurrent":true}]}
|
||||
currentVersion="0.2"
|
||||
fullVersion="0.2.86"
|
||||
versions={[{"version":"0.3","href":"/lume/reference/cli-reference","isCurrent":true},{"version":"0.2","href":"/lume/reference/v0.2/cli-reference","isCurrent":false}]}
|
||||
currentVersion="0.3"
|
||||
fullVersion="0.3.0"
|
||||
packageName="lume"
|
||||
installCommand="curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh | bash"
|
||||
/>
|
||||
|
||||
@@ -7,7 +7,7 @@ description: HTTP API reference for Lume server
|
||||
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
|
||||
Generated by: npx tsx scripts/docs-generators/lume.ts
|
||||
Source: libs/lume/src/Server/*.swift
|
||||
Version: 0.2.86
|
||||
Version: 0.3.0
|
||||
*/}
|
||||
|
||||
import { Callout } from 'fumadocs-ui/components/callout';
|
||||
@@ -15,9 +15,9 @@ import { Tabs, Tab } from 'fumadocs-ui/components/tabs';
|
||||
import { VersionHeader } from '@/components/version-selector';
|
||||
|
||||
<VersionHeader
|
||||
versions={[{"version":"0.2","href":"/lume/reference/cli-reference","isCurrent":true}]}
|
||||
currentVersion="0.2"
|
||||
fullVersion="0.2.86"
|
||||
versions={[{"version":"0.3","href":"/lume/reference/cli-reference","isCurrent":true},{"version":"0.2","href":"/lume/reference/v0.2/cli-reference","isCurrent":false}]}
|
||||
currentVersion="0.3"
|
||||
fullVersion="0.3.0"
|
||||
packageName="lume"
|
||||
installCommand="curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/lume/scripts/install.sh | bash"
|
||||
/>
|
||||
|
||||
@@ -1,21 +1,52 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# setup-cua-computer.sh
|
||||
# Makes a Lume macOS VM image cua-computer compatible
|
||||
# setup-cua.sh
|
||||
# Makes a Lume macOS VM fully cua-compatible
|
||||
#
|
||||
# This script runs ON the VM itself (not the host).
|
||||
# It installs cua-computer-server, cua-agent, Playwright, and configures
|
||||
# the VM for headless CUA operation (auto-login, disable sleep, LaunchAgent).
|
||||
# It installs cua-computer-server, cua-agent, Playwright, noVNC (web-based VNC
|
||||
# proxying to lume's built-in VNC on the host), supervisor, and configures the
|
||||
# VM for headless CUA operation (auto-login, disable sleep, LaunchAgent).
|
||||
#
|
||||
# In Lume VMs, the display is provided by the Virtualization Framework and
|
||||
# exposed through lume's built-in VNC server on the host. macOS Screen Sharing
|
||||
# inside the VM has no framebuffer to capture. Therefore, noVNC/websockify
|
||||
# inside the VM proxies to lume's VNC on the host (via the gateway IP).
|
||||
#
|
||||
# Usage:
|
||||
# lume ssh <vm-name> "curl -fsSL <url> | bash"
|
||||
# lume ssh <vm-name> < setup-cua-computer.sh
|
||||
# # Copy the script into the VM first, then run with a generous timeout:
|
||||
# lume ssh <vm-name> "cat > /tmp/setup.sh && chmod +x /tmp/setup.sh" < setup-cua.sh
|
||||
# lume ssh <vm-name> --timeout 600 "bash /tmp/setup.sh --yes"
|
||||
#
|
||||
# # Or copy and run directly inside the VM:
|
||||
# ./setup-cua-computer.sh [--yes] [--port PORT]
|
||||
# ./setup-cua.sh [--yes] [--port PORT] [--host-vnc-port PORT]
|
||||
#
|
||||
# Important: Start the VM with a fixed VNC port and password so websockify can find it:
|
||||
# lume run <vm-name> --vnc-port 5901 --vnc-password lume
|
||||
#
|
||||
# Note: Port 5900 is often used by the host's Screen Sharing service, so we
|
||||
# default to 5901. Use --host-vnc-port to match whatever --vnc-port you pass
|
||||
# to lume run.
|
||||
#
|
||||
# Accessing noVNC remotely (from outside the host):
|
||||
# 1. SSH tunnel: ssh -L 6080:<vm-ip>:6080 user@host
|
||||
# 2. Open: http://localhost:6080/vnc.html?host=localhost&port=6080
|
||||
# The ?host=localhost&port=6080 params ensure the websocket connects through
|
||||
# the tunnel instead of trying to reach the VM IP directly.
|
||||
#
|
||||
# Note: The default lume ssh timeout is 60s, which is too short for this script
|
||||
# (Homebrew + Xcode CLT + pip installs can take 5-10 minutes). Use --timeout 600
|
||||
# or --timeout 0 (unlimited) when running via lume ssh.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - macOS VM created via lume (with SSH enabled, user lume/lume)
|
||||
#
|
||||
# Host troubleshooting:
|
||||
# If VMs fail to start with "Unable to access security information", the host's
|
||||
# login keychain is likely locked (common after host reboot/crash on macOS Sequoia+).
|
||||
# Fix: security unlock-keychain -p '<host-password>' ~/Library/Keychains/login.keychain-db
|
||||
# Prevention: enable auto-login on the host so the keychain unlocks at boot.
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
@@ -33,6 +64,9 @@ AUTO_YES=false
|
||||
CUA_SERVER_PORT=8443
|
||||
VM_PASSWORD="${VM_PASSWORD:-lume}"
|
||||
CUA_DIR="$HOME/.cua-server"
|
||||
NOVNC_PORT=6080
|
||||
HOST_VNC_PORT="${HOST_VNC_PORT:-5901}"
|
||||
VNC_PASSWORD="${VNC_PASSWORD:-lume}"
|
||||
|
||||
# Parse arguments
|
||||
while [ "$#" -gt 0 ]; do
|
||||
@@ -44,13 +78,28 @@ while [ "$#" -gt 0 ]; do
|
||||
CUA_SERVER_PORT="$2"
|
||||
shift
|
||||
;;
|
||||
--host-vnc-port)
|
||||
HOST_VNC_PORT="$2"
|
||||
shift
|
||||
;;
|
||||
--vnc-password)
|
||||
VNC_PASSWORD="$2"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --yes, -y Non-interactive mode (accept all prompts)"
|
||||
echo " --port PORT CUA computer server port (default: 8443)"
|
||||
echo " --help Show this help message"
|
||||
echo " --yes, -y Non-interactive mode (accept all prompts)"
|
||||
echo " --port PORT CUA computer server port (default: 8443)"
|
||||
echo " --host-vnc-port PORT Lume VNC port on the host (default: 5901)"
|
||||
echo " --vnc-password PWD Lume VNC password (from 'lume ls' output)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment variables:"
|
||||
echo " VM_PASSWORD VM user password (default: lume)"
|
||||
echo " HOST_VNC_PORT Lume VNC port on the host (default: 5901)"
|
||||
echo " VNC_PASSWORD Lume VNC password (from 'lume ls' output)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
@@ -64,6 +113,12 @@ done
|
||||
CURRENT_USER="$(whoami)"
|
||||
USER_HOME="$HOME"
|
||||
|
||||
# Auto-detect host gateway IP
|
||||
GATEWAY_IP=$(route get default 2>/dev/null | grep gateway | awk '{print $2}')
|
||||
if [ -z "$GATEWAY_IP" ]; then
|
||||
GATEWAY_IP="192.168.64.1"
|
||||
fi
|
||||
|
||||
# ============================================
|
||||
# Helpers
|
||||
# ============================================
|
||||
@@ -174,7 +229,7 @@ install_system_deps() {
|
||||
ensure_brew
|
||||
|
||||
local deps_needed=()
|
||||
for pkg in ffmpeg; do
|
||||
for pkg in ffmpeg supervisor wget; do
|
||||
if ! brew list "$pkg" &>/dev/null; then
|
||||
deps_needed+=("$pkg")
|
||||
fi
|
||||
@@ -227,17 +282,94 @@ setup_cua_server() {
|
||||
pip install playwright && playwright install firefox || \
|
||||
echo -e "${YELLOW}Playwright install failed (non-blocking, BrowserTool may not work)${NC}"
|
||||
|
||||
# Install websockify for noVNC and vncdotool for VNC backend
|
||||
echo "Installing websockify and vncdotool..."
|
||||
pip install websockify vncdotool
|
||||
|
||||
deactivate
|
||||
|
||||
echo -e "${GREEN}CUA server and agent installed${NC}"
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# Phase 5: Startup Script + LaunchAgent
|
||||
# Phase 5: noVNC + Websockify
|
||||
# ============================================
|
||||
|
||||
setup_novnc() {
|
||||
print_phase "Phase 5: noVNC (Web-based VNC)"
|
||||
|
||||
if ! confirm "Install noVNC and websockify (web VNC on port $NOVNC_PORT)?"; then
|
||||
echo -e "${YELLOW}Skipping noVNC setup.${NC}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
ensure_brew
|
||||
|
||||
local NOVNC_DIR="/usr/local/share/novnc"
|
||||
|
||||
setup_sudo
|
||||
|
||||
# Download noVNC
|
||||
echo "Installing noVNC..."
|
||||
cd /tmp
|
||||
rm -rf noVNC-master noVNC-master.zip
|
||||
wget -q https://github.com/trycua/noVNC/archive/refs/heads/master.zip -O noVNC-master.zip
|
||||
unzip -q noVNC-master.zip
|
||||
|
||||
if [ "$AUTO_YES" = true ]; then
|
||||
sudo -A rm -rf "$NOVNC_DIR"
|
||||
sudo -A mkdir -p "$NOVNC_DIR"
|
||||
sudo -A mv noVNC-master/* "$NOVNC_DIR/"
|
||||
sudo -A ln -sf "$NOVNC_DIR/vnc.html" "$NOVNC_DIR/index.html"
|
||||
sudo -A chmod -R 755 "$NOVNC_DIR"
|
||||
else
|
||||
sudo rm -rf "$NOVNC_DIR"
|
||||
sudo mkdir -p "$NOVNC_DIR"
|
||||
sudo mv noVNC-master/* "$NOVNC_DIR/"
|
||||
sudo ln -sf "$NOVNC_DIR/vnc.html" "$NOVNC_DIR/index.html"
|
||||
sudo chmod -R 755 "$NOVNC_DIR"
|
||||
fi
|
||||
|
||||
rm -rf noVNC-master noVNC-master.zip
|
||||
|
||||
# Configure supervisor for websockify
|
||||
# Proxies to lume's built-in VNC on the host via the gateway IP.
|
||||
# The host VNC port must match the --vnc-port used with lume run.
|
||||
local SUPERVISOR_CONF_DIR="/opt/homebrew/etc/supervisor.d"
|
||||
local SUPERVISOR_LOG_DIR="/opt/homebrew/var/log/supervisor"
|
||||
local VENV_PYTHON="$CUA_DIR/venv/bin/python3"
|
||||
|
||||
mkdir -p "$SUPERVISOR_CONF_DIR"
|
||||
mkdir -p "$SUPERVISOR_LOG_DIR"
|
||||
|
||||
cat > "$SUPERVISOR_CONF_DIR/websockify.ini" <<EOF
|
||||
[program:websockify]
|
||||
command=$VENV_PYTHON -m websockify --web $NOVNC_DIR $NOVNC_PORT $GATEWAY_IP:$HOST_VNC_PORT
|
||||
directory=$NOVNC_DIR
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startsecs=5
|
||||
startretries=3
|
||||
stopwaitsecs=10
|
||||
redirect_stderr=true
|
||||
stdout_logfile=$SUPERVISOR_LOG_DIR/websockify.log
|
||||
stdout_logfile_maxbytes=10MB
|
||||
stdout_logfile_backups=5
|
||||
environment=HOME="$USER_HOME",PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin",DISPLAY=":0"
|
||||
EOF
|
||||
|
||||
chmod 644 "$SUPERVISOR_CONF_DIR/websockify.ini"
|
||||
|
||||
echo -e "${GREEN}noVNC installed${NC}"
|
||||
echo " websockify will proxy port $NOVNC_PORT -> $GATEWAY_IP:$HOST_VNC_PORT (lume VNC)"
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# Phase 6: Startup Script + LaunchAgent
|
||||
# ============================================
|
||||
|
||||
create_startup_script() {
|
||||
print_phase "Phase 5: Startup Script & LaunchAgent"
|
||||
print_phase "Phase 6: Startup Script & LaunchAgent"
|
||||
|
||||
if ! confirm "Create CUA server startup script and LaunchAgent?"; then
|
||||
echo -e "${YELLOW}Skipping LaunchAgent setup.${NC}"
|
||||
@@ -260,6 +392,48 @@ source venv/bin/activate
|
||||
|
||||
export DISPLAY=:0
|
||||
|
||||
# Auto-detect host gateway IP for VNC backend
|
||||
GATEWAY_IP=$(route get default 2>/dev/null | grep gateway | awk '{print $2}')
|
||||
if [ -z "$GATEWAY_IP" ]; then
|
||||
GATEWAY_IP="192.168.64.1"
|
||||
fi
|
||||
|
||||
# Read VNC config from lume shared directory if available.
|
||||
# Lume automatically shares a "lume-config" VirtioFS volume containing vnc.env
|
||||
# with the assigned VNC port and password for this VM.
|
||||
LUME_CONFIG_MOUNT="/tmp/lume-config"
|
||||
LUME_CONFIG="$LUME_CONFIG_MOUNT/vnc.env"
|
||||
LUME_VNC_PORT=""
|
||||
LUME_VNC_PASSWORD=""
|
||||
|
||||
# Try to mount the lume-config VirtioFS share and read VNC config
|
||||
mkdir -p "$LUME_CONFIG_MOUNT" 2>/dev/null
|
||||
if mount | grep -q "lume-config"; then
|
||||
echo "lume-config already mounted" >> "$LOG_FILE"
|
||||
elif mount_virtiofs lume-config "$LUME_CONFIG_MOUNT" 2>/dev/null; then
|
||||
echo "Mounted lume-config VirtioFS share" >> "$LOG_FILE"
|
||||
else
|
||||
echo "lume-config VirtioFS share not available, using defaults" >> "$LOG_FILE"
|
||||
fi
|
||||
|
||||
# Wait up to 10s for vnc.env to appear (written by host after VNC starts)
|
||||
for i in $(seq 1 10); do
|
||||
if [ -f "$LUME_CONFIG" ]; then
|
||||
eval "$(cat "$LUME_CONFIG")"
|
||||
LUME_VNC_PORT="${VNC_PORT:-}"
|
||||
LUME_VNC_PASSWORD="${VNC_PASSWORD:-}"
|
||||
echo "Read VNC config from lume shared dir: port=$LUME_VNC_PORT" >> "$LOG_FILE"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Set VNC backend env vars — prefer lume-config values, fall back to defaults
|
||||
export CUA_BACKEND=vnc
|
||||
export CUA_VNC_HOST="$GATEWAY_IP"
|
||||
export CUA_VNC_PORT="${LUME_VNC_PORT:-__VNC_PORT__}"
|
||||
export CUA_VNC_PASSWORD="${LUME_VNC_PASSWORD:-__VNC_PASSWORD__}"
|
||||
|
||||
# Update packages
|
||||
echo "Updating cua-agent..." >> "$LOG_FILE"
|
||||
pip install --upgrade --no-input "cua-agent[all]" >> "$LOG_FILE" 2>&1 || true
|
||||
@@ -272,13 +446,16 @@ echo "Ensuring Playwright Firefox..." >> "$LOG_FILE"
|
||||
pip install --upgrade --no-input playwright >> "$LOG_FILE" 2>&1 || true
|
||||
playwright install firefox >> "$LOG_FILE" 2>&1 || true
|
||||
|
||||
# Start server
|
||||
echo "Starting CUA computer server on port __PORT__..." >> "$LOG_FILE"
|
||||
# Start server with VNC backend pointing at lume's VNC on the host
|
||||
echo "Starting CUA computer server (vnc backend) on port __PORT__..." >> "$LOG_FILE"
|
||||
echo " VNC target: $GATEWAY_IP:__VNC_PORT__ (password: __VNC_PASSWORD__)" >> "$LOG_FILE"
|
||||
python -m computer_server --port __PORT__ >> "$LOG_FILE" 2>&1
|
||||
SCRIPT_EOF
|
||||
|
||||
# Stamp the port
|
||||
# Stamp the port and VNC settings
|
||||
sed -i '' "s/__PORT__/$CUA_SERVER_PORT/g" "$STARTUP_SCRIPT"
|
||||
sed -i '' "s/__VNC_PORT__/$HOST_VNC_PORT/g" "$STARTUP_SCRIPT"
|
||||
sed -i '' "s/__VNC_PASSWORD__/$VNC_PASSWORD/g" "$STARTUP_SCRIPT"
|
||||
chmod +x "$STARTUP_SCRIPT"
|
||||
|
||||
# Create LaunchAgent
|
||||
@@ -309,6 +486,12 @@ SCRIPT_EOF
|
||||
<string>$USER_HOME</string>
|
||||
<key>DISPLAY</key>
|
||||
<string>:0</string>
|
||||
<key>CUA_BACKEND</key>
|
||||
<string>vnc</string>
|
||||
<key>CUA_VNC_PORT</key>
|
||||
<string>$HOST_VNC_PORT</string>
|
||||
<key>CUA_VNC_PASSWORD</key>
|
||||
<string>$VNC_PASSWORD</string>
|
||||
</dict>
|
||||
|
||||
<key>ProgramArguments</key>
|
||||
@@ -346,13 +529,13 @@ EOF
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# Phase 6: System Configuration
|
||||
# Phase 7: System Configuration + Auto-Login
|
||||
# ============================================
|
||||
|
||||
configure_system() {
|
||||
print_phase "Phase 6: System Configuration"
|
||||
print_phase "Phase 7: System Configuration"
|
||||
|
||||
if ! confirm "Disable screen lock, screensaver, and sleep?"; then
|
||||
if ! confirm "Disable screen lock, screensaver, sleep, and enable auto-login?"; then
|
||||
echo -e "${YELLOW}Skipping system configuration.${NC}"
|
||||
return 0
|
||||
fi
|
||||
@@ -374,11 +557,67 @@ configure_system() {
|
||||
# Enable full keyboard navigation (needed for accessibility automation)
|
||||
defaults write NSGlobalDomain AppleKeyboardUIMode -int 3
|
||||
|
||||
echo -e "${GREEN}System configured (no sleep, no screensaver, full keyboard nav)${NC}"
|
||||
# Configure auto-login (ensures GUI session + unlocked keychain on boot)
|
||||
echo "Configuring auto-login for $CURRENT_USER..."
|
||||
|
||||
if [ "$AUTO_YES" = true ]; then
|
||||
sudo -A defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser "$CURRENT_USER"
|
||||
else
|
||||
sudo defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser "$CURRENT_USER"
|
||||
fi
|
||||
|
||||
# Create kcpassword file (XOR-encrypted password for auto-login)
|
||||
python3 << PYEOF
|
||||
password = "$VM_PASSWORD"
|
||||
key = bytes.fromhex("7d895223d2bcddeaa3b91f")
|
||||
padded_len = ((len(password) + 11) // 12) * 12
|
||||
padded = password + "\x00" * (padded_len - len(password))
|
||||
result = bytearray()
|
||||
for i, char in enumerate(padded.encode("utf-8")):
|
||||
result.append(char ^ key[i % len(key)])
|
||||
with open("/tmp/kcpassword", "wb") as f:
|
||||
f.write(result)
|
||||
PYEOF
|
||||
|
||||
if [ "$AUTO_YES" = true ]; then
|
||||
sudo -A mv /tmp/kcpassword /etc/kcpassword
|
||||
sudo -A chown root:wheel /etc/kcpassword
|
||||
sudo -A chmod 600 /etc/kcpassword
|
||||
else
|
||||
sudo mv /tmp/kcpassword /etc/kcpassword
|
||||
sudo chown root:wheel /etc/kcpassword
|
||||
sudo chmod 600 /etc/kcpassword
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}System configured (no sleep, no screensaver, full keyboard nav, auto-login)${NC}"
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# Phase 7: Verification
|
||||
# Phase 8: Start Services
|
||||
# ============================================
|
||||
|
||||
start_services() {
|
||||
print_phase "Phase 8: Start Services"
|
||||
|
||||
ensure_brew
|
||||
|
||||
# Start supervisor (manages websockify)
|
||||
if command -v brew &>/dev/null; then
|
||||
echo "Starting supervisor..."
|
||||
brew services start supervisor 2>/dev/null || true
|
||||
sleep 3
|
||||
|
||||
if command -v supervisorctl &>/dev/null; then
|
||||
supervisorctl reread 2>/dev/null || true
|
||||
supervisorctl update 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}Services started${NC}"
|
||||
}
|
||||
|
||||
# ============================================
|
||||
# Phase 9: Verification
|
||||
# ============================================
|
||||
|
||||
verify() {
|
||||
@@ -386,6 +625,7 @@ verify() {
|
||||
|
||||
echo "System:"
|
||||
echo " User: $CURRENT_USER"
|
||||
echo " Gateway: $GATEWAY_IP"
|
||||
sw_vers | sed 's/^/ /'
|
||||
|
||||
echo ""
|
||||
@@ -431,6 +671,22 @@ verify() {
|
||||
echo -e " ${YELLOW}~${NC} Playwright (may not be installed)"
|
||||
fi
|
||||
|
||||
# noVNC / websockify
|
||||
if command -v supervisorctl &>/dev/null && supervisorctl status websockify 2>/dev/null | grep -q RUNNING; then
|
||||
echo -e " ${GREEN}✓${NC} noVNC + websockify (port $NOVNC_PORT -> $GATEWAY_IP:$HOST_VNC_PORT)"
|
||||
elif [ -d "/usr/local/share/novnc" ]; then
|
||||
echo -e " ${YELLOW}~${NC} noVNC installed (websockify may not be running yet)"
|
||||
else
|
||||
echo -e " ${RED}✗${NC} noVNC"
|
||||
fi
|
||||
|
||||
# Auto-login
|
||||
if defaults read /Library/Preferences/com.apple.loginwindow autoLoginUser 2>/dev/null | grep -q "$CURRENT_USER"; then
|
||||
echo -e " ${GREEN}✓${NC} Auto-login ($CURRENT_USER)"
|
||||
else
|
||||
echo -e " ${YELLOW}~${NC} Auto-login (may need reboot)"
|
||||
fi
|
||||
|
||||
# LaunchAgent
|
||||
local PLIST="$USER_HOME/Library/LaunchAgents/com.trycua.computer_server.plist"
|
||||
if [ -f "$PLIST" ]; then
|
||||
@@ -444,13 +700,29 @@ verify() {
|
||||
echo -e "${GREEN} Setup Complete!${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
echo "CUA computer server will start on port $CUA_SERVER_PORT at next login."
|
||||
echo "Services:"
|
||||
echo " CUA server: port $CUA_SERVER_PORT (starts at login via LaunchAgent)"
|
||||
echo " noVNC: port $NOVNC_PORT -> $GATEWAY_IP:$HOST_VNC_PORT (lume VNC)"
|
||||
echo ""
|
||||
echo "To start it now (from the host):"
|
||||
echo "Start the VM with a fixed VNC port and password (from the host):"
|
||||
echo " lume run <vm-name> --vnc-port $HOST_VNC_PORT --vnc-password $VNC_PASSWORD"
|
||||
echo ""
|
||||
echo "To start CUA server now (from the host):"
|
||||
echo " lume ssh <vm-name> 'launchctl load ~/Library/LaunchAgents/com.trycua.computer_server.plist'"
|
||||
echo ""
|
||||
echo "To test from the host:"
|
||||
echo " curl http://<vm-ip>:$CUA_SERVER_PORT/health"
|
||||
echo " curl http://<vm-ip>:$CUA_SERVER_PORT/status"
|
||||
echo ""
|
||||
echo "To access noVNC from the host network:"
|
||||
echo " http://<vm-ip>:$NOVNC_PORT/vnc.html"
|
||||
echo ""
|
||||
echo "To access noVNC remotely (from outside the host):"
|
||||
echo " 1. SSH tunnel: ssh -L $NOVNC_PORT:<vm-ip>:$NOVNC_PORT user@host"
|
||||
echo " 2. Open: http://localhost:$NOVNC_PORT/vnc.html?host=localhost&port=$NOVNC_PORT"
|
||||
echo ""
|
||||
echo "VNC password: $VNC_PASSWORD"
|
||||
echo ""
|
||||
echo "NOTE: A reboot is recommended for auto-login to take effect."
|
||||
echo ""
|
||||
}
|
||||
|
||||
@@ -463,14 +735,16 @@ echo " ⠀⣀⣀⡀⠀⠀⠀⠀⢀⣀⣀⣀⡀⠘⠋⢉⠙⣷⠀⠀ ⠀"
|
||||
echo " ⠀⠀⢀⣴⣿⡿⠋⣉⠁⣠⣾⣿⣿⣿⣿⡿⠿⣦⡈⠀⣿⡇⠃⠀"
|
||||
echo " ⠀⠀⠀⣽⣿⣧⠀⠃⢰⣿⣿⡏⠙⣿⠿⢧⣀⣼⣷⠀⡿⠃⠀⠀"
|
||||
echo " ⠀⠀⠀⠉⣿⣿⣦⠀⢿⣿⣿⣷⣾⡏⠀⠀⢹⣿⣿⠀⠀⠀⠀⠀⠀"
|
||||
echo -e " ⠀⠀⠀⠀⠀⠉⠛⠁⠈⠿⣿⣿⣿⣷⣄⣠⡼⠟⠁${NC} ${BLUE}CUA Computer Setup${NC}"
|
||||
echo -e "${BLUE} Make a Lume VM cua-computer compatible${NC}"
|
||||
echo -e " ⠀⠀⠀⠀⠀⠉⠛⠁⠈⠿⣿⣿⣿⣷⣄⣠⡼⠟⠁${NC} ${BLUE}CUA Setup${NC}"
|
||||
echo -e "${BLUE} Make a Lume VM cua-compatible${NC}"
|
||||
echo ""
|
||||
|
||||
install_homebrew
|
||||
install_python
|
||||
install_system_deps
|
||||
setup_cua_server
|
||||
setup_novnc
|
||||
create_startup_script
|
||||
configure_system
|
||||
start_services
|
||||
verify
|
||||
@@ -45,6 +45,11 @@ struct Run: AsyncParsableCommand {
|
||||
help: "Port to use for the VNC server. Defaults to 0 (auto-assign)")
|
||||
var vncPort: Int = 0
|
||||
|
||||
@Option(
|
||||
name: [.customLong("vnc-password")],
|
||||
help: "Password for the VNC server. Defaults to a random passphrase")
|
||||
var vncPassword: String?
|
||||
|
||||
@Option(help: "For MacOS VMs only, boot into the VM in recovery mode")
|
||||
var recoveryMode: Bool = false
|
||||
|
||||
@@ -133,6 +138,7 @@ struct Run: AsyncParsableCommand {
|
||||
registry: registry,
|
||||
organization: organization,
|
||||
vncPort: vncPort,
|
||||
vncPassword: vncPassword,
|
||||
recoveryMode: recoveryMode,
|
||||
storage: storage,
|
||||
usbMassStoragePaths: parsedUSBStorageDevices.isEmpty ? nil : parsedUSBStorageDevices,
|
||||
|
||||
@@ -904,6 +904,7 @@ final class LumeController {
|
||||
registry: String = "ghcr.io",
|
||||
organization: String = "trycua",
|
||||
vncPort: Int = 0,
|
||||
vncPassword: String? = nil,
|
||||
recoveryMode: Bool = false,
|
||||
storage: String? = nil,
|
||||
usbMassStoragePaths: [Path]? = nil,
|
||||
@@ -1004,6 +1005,7 @@ final class LumeController {
|
||||
sharedDirectories: sharedDirectories,
|
||||
mount: mount,
|
||||
vncPort: vncPort,
|
||||
vncPassword: vncPassword,
|
||||
recoveryMode: recoveryMode,
|
||||
usbMassStoragePaths: usbMassStoragePaths,
|
||||
networkMode: networkMode,
|
||||
|
||||
@@ -136,7 +136,7 @@ class VM {
|
||||
|
||||
func run(
|
||||
noDisplay: Bool, sharedDirectories: [SharedDirectory], mount: Path?, vncPort: Int = 0,
|
||||
recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,
|
||||
vncPassword: String? = nil, recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,
|
||||
networkMode: NetworkMode? = nil, clipboard: Bool = false
|
||||
) async throws {
|
||||
Logger.info(
|
||||
@@ -215,13 +215,29 @@ class VM {
|
||||
|
||||
// Create and configure the VM
|
||||
do {
|
||||
// Create a lume-config shared directory so the guest can discover
|
||||
// the VNC port/password at boot. The directory is created empty now
|
||||
// and populated after the VNC server starts (VirtioFS exposes live
|
||||
// host directory contents, so the guest will see the file once written).
|
||||
let lumeConfigDir = FileManager.default.temporaryDirectory
|
||||
.appendingPathComponent("lume-config-\(vmDirContext.name)")
|
||||
try? FileManager.default.createDirectory(at: lumeConfigDir, withIntermediateDirectories: true)
|
||||
// Remove stale vnc.env from a previous run so the guest doesn't
|
||||
// read outdated port/password before the new file is written.
|
||||
try? FileManager.default.removeItem(
|
||||
at: lumeConfigDir.appendingPathComponent("vnc.env"))
|
||||
let lumeConfigSharedDir = SharedDirectory(
|
||||
hostPath: lumeConfigDir.path, tag: "lume-config", readOnly: true)
|
||||
var allSharedDirectories = sharedDirectories
|
||||
allSharedDirectories.append(lumeConfigSharedDir)
|
||||
|
||||
Logger.info(
|
||||
"Creating virtualization service context", metadata: ["name": vmDirContext.name])
|
||||
let config = try createVMVirtualizationServiceContext(
|
||||
cpuCount: cpuCount,
|
||||
memorySize: memorySize,
|
||||
display: vmDirContext.config.display.string,
|
||||
sharedDirectories: sharedDirectories,
|
||||
sharedDirectories: allSharedDirectories,
|
||||
mount: mount,
|
||||
recoveryMode: recoveryMode,
|
||||
usbMassStoragePaths: usbMassStoragePaths,
|
||||
@@ -246,7 +262,21 @@ class VM {
|
||||
"port": "\(vncPort)",
|
||||
])
|
||||
let vncInfo = try await setupSession(
|
||||
port: vncPort, sharedDirectories: sharedDirectories)
|
||||
port: vncPort, password: vncPassword, sharedDirectories: sharedDirectories)
|
||||
|
||||
// Write VNC connection info to the shared config directory so the
|
||||
// guest can read it via mount_virtiofs.
|
||||
// URL format: vnc://:password@host:port — URLComponents needs http:// to parse correctly.
|
||||
if let components = URLComponents(string: vncInfo.replacingOccurrences(of: "vnc://", with: "http://")),
|
||||
let port = components.port {
|
||||
let password = components.password ?? ""
|
||||
let envContent = "VNC_PORT=\(port)\nVNC_PASSWORD=\(password)\n"
|
||||
try? envContent.write(
|
||||
to: lumeConfigDir.appendingPathComponent("vnc.env"),
|
||||
atomically: true, encoding: .utf8)
|
||||
Logger.info("Wrote VNC config to shared directory", metadata: [
|
||||
"port": "\(port)", "path": lumeConfigDir.path])
|
||||
}
|
||||
Logger.info(
|
||||
"VNC setup successful", metadata: ["name": vmDirContext.name, "vncInfo": vncInfo])
|
||||
|
||||
@@ -658,12 +688,12 @@ class VM {
|
||||
}
|
||||
|
||||
/// Sets up the VNC service and returns the VNC URL
|
||||
private func startVNCService(port: Int = 0) async throws -> String {
|
||||
private func startVNCService(port: Int = 0, password: String? = nil) async throws -> String {
|
||||
guard let service = virtualizationService else {
|
||||
throw VMError.internalError("Virtualization service not initialized")
|
||||
}
|
||||
|
||||
try await vncService.start(port: port, virtualMachine: service.getVirtualMachine())
|
||||
try await vncService.start(port: port, password: password, virtualMachine: service.getVirtualMachine())
|
||||
|
||||
guard let url = vncService.url else {
|
||||
throw VMError.vncNotConfigured
|
||||
@@ -692,10 +722,10 @@ class VM {
|
||||
|
||||
/// Main session setup method that handles VNC and persists session data
|
||||
private func setupSession(
|
||||
port: Int = 0, sharedDirectories: [SharedDirectory] = []
|
||||
port: Int = 0, password: String? = nil, sharedDirectories: [SharedDirectory] = []
|
||||
) async throws -> String {
|
||||
// Start the VNC service and get the URL
|
||||
let url = try await startVNCService(port: port)
|
||||
let url = try await startVNCService(port: port, password: password)
|
||||
|
||||
// Save the session data
|
||||
saveSessionData(url: url, sharedDirectories: sharedDirectories)
|
||||
@@ -901,11 +931,22 @@ class VM {
|
||||
|
||||
// Create and configure the VM
|
||||
do {
|
||||
// Create lume-config shared directory for VNC discovery
|
||||
let lumeConfigDir = FileManager.default.temporaryDirectory
|
||||
.appendingPathComponent("lume-config-\(vmDirContext.name)")
|
||||
try? FileManager.default.createDirectory(at: lumeConfigDir, withIntermediateDirectories: true)
|
||||
try? FileManager.default.removeItem(
|
||||
at: lumeConfigDir.appendingPathComponent("vnc.env"))
|
||||
let lumeConfigSharedDir = SharedDirectory(
|
||||
hostPath: lumeConfigDir.path, tag: "lume-config", readOnly: true)
|
||||
var allSharedDirectories = sharedDirectories
|
||||
allSharedDirectories.append(lumeConfigSharedDir)
|
||||
|
||||
let config = try createVMVirtualizationServiceContext(
|
||||
cpuCount: cpuCount,
|
||||
memorySize: memorySize,
|
||||
display: vmDirContext.config.display.string,
|
||||
sharedDirectories: sharedDirectories,
|
||||
sharedDirectories: allSharedDirectories,
|
||||
mount: mount,
|
||||
recoveryMode: recoveryMode,
|
||||
usbMassStoragePaths: usbImagePaths
|
||||
@@ -916,6 +957,16 @@ class VM {
|
||||
port: vncPort, sharedDirectories: sharedDirectories)
|
||||
Logger.info("VNC info", metadata: ["vncInfo": vncInfo])
|
||||
|
||||
// Write VNC config to shared directory for guest discovery
|
||||
if let components = URLComponents(string: vncInfo.replacingOccurrences(of: "vnc://", with: "http://")),
|
||||
let port = components.port {
|
||||
let password = components.password ?? ""
|
||||
let envContent = "VNC_PORT=\(port)\nVNC_PASSWORD=\(password)\n"
|
||||
try? envContent.write(
|
||||
to: lumeConfigDir.appendingPathComponent("vnc.env"),
|
||||
atomically: true, encoding: .utf8)
|
||||
}
|
||||
|
||||
// Start the VM
|
||||
guard let service = virtualizationService else {
|
||||
throw VMError.internalError("Virtualization service not initialized")
|
||||
|
||||
@@ -8,7 +8,7 @@ import Network
|
||||
@MainActor
|
||||
protocol VNCService {
|
||||
var url: String? { get }
|
||||
func start(port: Int, virtualMachine: Any?) async throws
|
||||
func start(port: Int, password: String?, virtualMachine: Any?) async throws
|
||||
func stop()
|
||||
func openClient(url: String) async throws
|
||||
|
||||
@@ -69,8 +69,8 @@ final class DefaultVNCService: VNCService {
|
||||
}
|
||||
}
|
||||
|
||||
func start(port: Int, virtualMachine: Any?) async throws {
|
||||
let password = Array(PassphraseGenerator().prefix(4)).joined(separator: "-")
|
||||
func start(port: Int, password: String? = nil, virtualMachine: Any?) async throws {
|
||||
let password = password ?? Array(PassphraseGenerator().prefix(4)).joined(separator: "-")
|
||||
let securityConfiguration = Dynamic._VZVNCAuthenticationSecurityConfiguration(password: password)
|
||||
|
||||
// Store password for later VNC client connection
|
||||
|
||||
@@ -23,13 +23,13 @@ class MockVM: VM {
|
||||
|
||||
override func run(
|
||||
noDisplay: Bool, sharedDirectories: [SharedDirectory], mount: Path?, vncPort: Int = 0,
|
||||
recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,
|
||||
vncPassword: String? = nil, recoveryMode: Bool = false, usbMassStoragePaths: [Path]? = nil,
|
||||
networkMode: NetworkMode? = nil, clipboard: Bool = false
|
||||
) async throws {
|
||||
mockIsRunning = true
|
||||
try await super.run(
|
||||
noDisplay: noDisplay, sharedDirectories: sharedDirectories, mount: mount,
|
||||
vncPort: vncPort, recoveryMode: recoveryMode,
|
||||
vncPort: vncPort, vncPassword: vncPassword, recoveryMode: recoveryMode,
|
||||
usbMassStoragePaths: usbMassStoragePaths,
|
||||
networkMode: networkMode, clipboard: clipboard
|
||||
)
|
||||
|
||||
@@ -29,7 +29,7 @@ final class MockVNCService: VNCService {
|
||||
return nil
|
||||
}
|
||||
|
||||
func start(port: Int, virtualMachine: Any?) async throws {
|
||||
func start(port: Int, password: String? = nil, virtualMachine: Any?) async throws {
|
||||
isRunning = true
|
||||
url = "vnc://localhost:\(port)"
|
||||
_attachedVM = virtualMachine
|
||||
|
||||
@@ -120,7 +120,9 @@ def main() -> None:
|
||||
os.environ["CUA_BACKEND"] = "vnc"
|
||||
if args.vnc_host:
|
||||
os.environ["CUA_VNC_HOST"] = args.vnc_host
|
||||
os.environ["CUA_VNC_PORT"] = str(args.vnc_port)
|
||||
# Only override env vars from CLI args when explicitly provided
|
||||
if args.vnc_port != 5900 or "CUA_VNC_PORT" not in os.environ:
|
||||
os.environ["CUA_VNC_PORT"] = str(args.vnc_port)
|
||||
if args.vnc_password:
|
||||
os.environ["CUA_VNC_PASSWORD"] = args.vnc_password
|
||||
vnc_host = args.vnc_host or os.environ.get("CUA_VNC_HOST")
|
||||
|
||||
@@ -21,7 +21,6 @@ Usage:
|
||||
import asyncio
|
||||
import base64
|
||||
import logging
|
||||
import threading
|
||||
from io import BytesIO
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -81,7 +80,12 @@ def _translate_key(key: str) -> str:
|
||||
|
||||
|
||||
class _VNCConnection:
|
||||
"""Manages a vncdotool ThreadedVNCClientProxy lifecycle.
|
||||
"""Manages vncdotool client connections to a VNC server.
|
||||
|
||||
Uses fresh connections per operation to avoid a bug in vncdotool's
|
||||
ThreadedVNCClientProxy where the Twisted deferred chain corrupts
|
||||
coordinate state after operations like refreshScreen. Each public
|
||||
method creates its own connection, executes, and disconnects.
|
||||
|
||||
All methods are synchronous (blocking) — the handler calls them via
|
||||
asyncio.to_thread to avoid blocking the event loop.
|
||||
@@ -91,164 +95,272 @@ class _VNCConnection:
|
||||
self._host = host
|
||||
self._port = port
|
||||
self._password = password
|
||||
self._client = None
|
||||
self._lock = threading.Lock()
|
||||
# Track cursor position locally (vncdotool also tracks via client.x/y)
|
||||
# Track cursor position locally
|
||||
self._cursor_x: int = 0
|
||||
self._cursor_y: int = 0
|
||||
|
||||
def _ensure_connected(self):
|
||||
"""Lazily connect to the VNC server. Returns the vncdotool client."""
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
with self._lock:
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
from vncdotool import api
|
||||
def _with_client(self, fn):
|
||||
"""Execute fn(client) with a fresh VNC connection via the global Twisted reactor.
|
||||
|
||||
server_str = f"{self._host}::{self._port}"
|
||||
logger.info(f"VNC connecting to {self._host}:{self._port}")
|
||||
client = api.connect(
|
||||
server_str,
|
||||
password=self._password or None,
|
||||
Starts the reactor in a background thread on first use, then reuses it.
|
||||
``fn(client)`` receives a raw ``VNCDoToolClient`` whose methods return
|
||||
Twisted Deferreds.
|
||||
"""
|
||||
import threading
|
||||
|
||||
from twisted.internet import defer, reactor
|
||||
from vncdotool.client import VNCDoToolFactory
|
||||
|
||||
result_holder: list = [None]
|
||||
error_holder: list = [None]
|
||||
done_event = threading.Event()
|
||||
|
||||
def _work():
|
||||
factory = VNCDoToolFactory()
|
||||
factory.password = self._password or None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _do():
|
||||
client = None
|
||||
try:
|
||||
reactor.connectTCP(self._host, self._port, factory)
|
||||
client = yield factory.deferred
|
||||
res = yield defer.maybeDeferred(fn, client)
|
||||
result_holder[0] = res
|
||||
except Exception as e:
|
||||
error_holder[0] = e
|
||||
finally:
|
||||
if client is not None and hasattr(client, "transport") and client.transport:
|
||||
client.transport.loseConnection()
|
||||
done_event.set()
|
||||
|
||||
_do()
|
||||
|
||||
# Ensure the reactor is running in a background thread
|
||||
if not reactor.running:
|
||||
t = threading.Thread(
|
||||
target=reactor.run,
|
||||
kwargs={"installSignalHandlers": False},
|
||||
daemon=True,
|
||||
)
|
||||
client.timeout = 30
|
||||
self._client = client
|
||||
logger.info("VNC connected")
|
||||
return client
|
||||
t.start()
|
||||
|
||||
reactor.callFromThread(_work)
|
||||
done_event.wait(timeout=30)
|
||||
if not done_event.is_set():
|
||||
raise TimeoutError("VNC operation timed out")
|
||||
if error_holder[0] is not None:
|
||||
raise error_holder[0]
|
||||
return result_holder[0]
|
||||
|
||||
def disconnect(self):
|
||||
with self._lock:
|
||||
if self._client is not None:
|
||||
try:
|
||||
self._client.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
self._client = None
|
||||
pass # No persistent connection to close
|
||||
|
||||
def _reset_on_error(self):
|
||||
"""Disconnect so next call reconnects."""
|
||||
try:
|
||||
self.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
pass # No persistent connection to reset
|
||||
|
||||
# -- Screenshot ---------------------------------------------------------
|
||||
|
||||
def capture_screenshot(self) -> bytes:
|
||||
"""Capture the screen and return PNG bytes."""
|
||||
client = self._ensure_connected()
|
||||
# refreshScreen updates the internal screen image
|
||||
client.refreshScreen(incremental=False)
|
||||
screen = client.protocol.screen
|
||||
buf = BytesIO()
|
||||
screen.save(buf, format="PNG")
|
||||
return buf.getvalue()
|
||||
from twisted.internet import defer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _do(client):
|
||||
yield client.refreshScreen(incremental=False)
|
||||
screen = client.screen
|
||||
buf = BytesIO()
|
||||
screen.save(buf, format="PNG")
|
||||
defer.returnValue(buf.getvalue())
|
||||
|
||||
return self._with_client(_do)
|
||||
|
||||
# -- Mouse --------------------------------------------------------------
|
||||
|
||||
def mouse_move(self, x: int, y: int):
|
||||
client = self._ensure_connected()
|
||||
client.mouseMove(x, y)
|
||||
self._with_client(lambda c: c.mouseMove(x, y))
|
||||
self._cursor_x = x
|
||||
self._cursor_y = y
|
||||
|
||||
def mouse_click(self, x: int, y: int, button: int = 1, clicks: int = 1):
|
||||
client = self._ensure_connected()
|
||||
client.mouseMove(x, y)
|
||||
from twisted.internet import defer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _do(client):
|
||||
yield client.mouseMove(x, y)
|
||||
for _ in range(clicks):
|
||||
yield client.mousePress(button)
|
||||
|
||||
self._with_client(_do)
|
||||
self._cursor_x = x
|
||||
self._cursor_y = y
|
||||
for _ in range(clicks):
|
||||
client.mousePress(button)
|
||||
|
||||
def mouse_down(self, x: int, y: int, button: int = 1):
|
||||
client = self._ensure_connected()
|
||||
client.mouseMove(x, y)
|
||||
from twisted.internet import defer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _do(client):
|
||||
yield client.mouseMove(x, y)
|
||||
yield client.mouseDown(button)
|
||||
|
||||
self._with_client(_do)
|
||||
self._cursor_x = x
|
||||
self._cursor_y = y
|
||||
client.mouseDown(button)
|
||||
|
||||
def mouse_up(self, x: int, y: int, button: int = 1):
|
||||
client = self._ensure_connected()
|
||||
client.mouseMove(x, y)
|
||||
from twisted.internet import defer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _do(client):
|
||||
yield client.mouseMove(x, y)
|
||||
yield client.mouseUp(button)
|
||||
|
||||
self._with_client(_do)
|
||||
self._cursor_x = x
|
||||
self._cursor_y = y
|
||||
client.mouseUp(button)
|
||||
|
||||
def mouse_drag(self, x: int, y: int, step: int = 1):
|
||||
client = self._ensure_connected()
|
||||
client.mouseDrag(x, y, step=step)
|
||||
from twisted.internet import defer
|
||||
|
||||
sx, sy = self._cursor_x, self._cursor_y
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _do(client):
|
||||
# Move in increments from current position to target
|
||||
dx, dy = x - sx, y - sy
|
||||
steps = max(abs(dx), abs(dy)) // max(step, 1)
|
||||
steps = max(steps, 1)
|
||||
yield client.mouseDown(1)
|
||||
for i in range(1, steps + 1):
|
||||
ix = sx + dx * i // steps
|
||||
iy = sy + dy * i // steps
|
||||
yield client.mouseMove(ix, iy)
|
||||
yield client.mouseUp(1)
|
||||
|
||||
self._with_client(_do)
|
||||
self._cursor_x = x
|
||||
self._cursor_y = y
|
||||
|
||||
def drag_to(
|
||||
self, start_x: int, start_y: int, end_x: int, end_y: int, button: int = 1, step: int = 5
|
||||
):
|
||||
"""Drag from start to end on a single connection."""
|
||||
from twisted.internet import defer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _do(client):
|
||||
yield client.mouseMove(start_x, start_y)
|
||||
yield client.mouseDown(button)
|
||||
# Manual drag in increments instead of mouseDrag (avoids doPoll)
|
||||
dx, dy = end_x - start_x, end_y - start_y
|
||||
steps = max(abs(dx), abs(dy)) // max(step, 1)
|
||||
steps = max(steps, 1)
|
||||
for i in range(1, steps + 1):
|
||||
ix = start_x + dx * i // steps
|
||||
iy = start_y + dy * i // steps
|
||||
yield client.mouseMove(ix, iy)
|
||||
yield client.mouseUp(button)
|
||||
|
||||
self._with_client(_do)
|
||||
self._cursor_x = end_x
|
||||
self._cursor_y = end_y
|
||||
|
||||
def drag_path(self, path: List[Tuple[int, int]], button: int = 1):
|
||||
"""Drag along a path of points on a single connection."""
|
||||
from twisted.internet import defer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _do(client):
|
||||
yield client.mouseMove(path[0][0], path[0][1])
|
||||
yield client.mouseDown(button)
|
||||
for px, py in path[1:]:
|
||||
yield client.mouseMove(px, py)
|
||||
yield client.mouseUp(button)
|
||||
|
||||
self._with_client(_do)
|
||||
self._cursor_x = path[-1][0]
|
||||
self._cursor_y = path[-1][1]
|
||||
|
||||
def scroll(self, x: int, y: int):
|
||||
"""Scroll. y>0 = down (button 5), y<0 = up (button 4)."""
|
||||
client = self._ensure_connected()
|
||||
if y > 0:
|
||||
for _ in range(y):
|
||||
client.mousePress(5)
|
||||
elif y < 0:
|
||||
for _ in range(abs(y)):
|
||||
client.mousePress(4)
|
||||
# Horizontal: button 6 (right), button 7 (left) — not universally supported
|
||||
if x > 0:
|
||||
for _ in range(x):
|
||||
client.mousePress(6)
|
||||
elif x < 0:
|
||||
for _ in range(abs(x)):
|
||||
client.mousePress(7)
|
||||
"""Scroll. y>0 = up, y<0 = down (matches macOS native handler convention).
|
||||
|
||||
Uses arrow key presses instead of mouse buttons 4/5 because Apple's
|
||||
_VZVNCServer (used by Lume) does not translate RFB mouse buttons 4-7
|
||||
into scroll wheel events.
|
||||
"""
|
||||
from twisted.internet import defer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _do(client):
|
||||
if y > 0:
|
||||
for _ in range(y):
|
||||
yield client.keyPress("up")
|
||||
elif y < 0:
|
||||
for _ in range(abs(y)):
|
||||
yield client.keyPress("down")
|
||||
if x > 0:
|
||||
for _ in range(x):
|
||||
yield client.keyPress("right")
|
||||
elif x < 0:
|
||||
for _ in range(abs(x)):
|
||||
yield client.keyPress("left")
|
||||
|
||||
self._with_client(_do)
|
||||
|
||||
# -- Keyboard -----------------------------------------------------------
|
||||
|
||||
def key_press(self, key: str):
|
||||
client = self._ensure_connected()
|
||||
client.keyPress(_translate_key(key))
|
||||
self._with_client(lambda c: c.keyPress(_translate_key(key)))
|
||||
|
||||
def key_down(self, key: str):
|
||||
client = self._ensure_connected()
|
||||
client.keyDown(_translate_key(key))
|
||||
self._with_client(lambda c: c.keyDown(_translate_key(key)))
|
||||
|
||||
def key_up(self, key: str):
|
||||
client = self._ensure_connected()
|
||||
client.keyUp(_translate_key(key))
|
||||
self._with_client(lambda c: c.keyUp(_translate_key(key)))
|
||||
|
||||
def type_text(self, text: str):
|
||||
"""Type text character by character, handling shift for uppercase/symbols."""
|
||||
client = self._ensure_connected()
|
||||
for ch in text:
|
||||
if ch == " ":
|
||||
client.keyPress("space")
|
||||
elif ch == "\n":
|
||||
client.keyPress("enter")
|
||||
elif ch == "\t":
|
||||
client.keyPress("tab")
|
||||
else:
|
||||
# vncdotool's keyPress handles single characters directly
|
||||
client.keyPress(ch)
|
||||
from twisted.internet import defer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _do(client):
|
||||
for ch in text:
|
||||
if ch == " ":
|
||||
yield client.keyPress("space")
|
||||
elif ch == "\n":
|
||||
yield client.keyPress("enter")
|
||||
elif ch == "\t":
|
||||
yield client.keyPress("tab")
|
||||
else:
|
||||
yield client.keyPress(ch)
|
||||
|
||||
self._with_client(_do)
|
||||
|
||||
def hotkey(self, keys: List[str]):
|
||||
"""Press a key combination (e.g. ['command', 'a'])."""
|
||||
client = self._ensure_connected()
|
||||
translated = [_translate_key(k) for k in keys]
|
||||
# vncdotool supports "modifier-key" syntax: "alt-a", "ctrl-shift-c"
|
||||
combo = "-".join(translated)
|
||||
client.keyPress(combo)
|
||||
self._with_client(lambda c: c.keyPress(combo))
|
||||
|
||||
# -- Clipboard ----------------------------------------------------------
|
||||
|
||||
def paste_text(self, text: str):
|
||||
client = self._ensure_connected()
|
||||
client.paste(text)
|
||||
self._with_client(lambda c: c.paste(text))
|
||||
|
||||
# -- Info ---------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def screen_size(self) -> Tuple[int, int]:
|
||||
client = self._ensure_connected()
|
||||
proto = client.protocol
|
||||
if proto is not None and proto.screen is not None:
|
||||
return proto.screen.size
|
||||
return (0, 0)
|
||||
from twisted.internet import defer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _do(client):
|
||||
yield client.refreshScreen(incremental=False)
|
||||
if client.screen is not None:
|
||||
defer.returnValue(client.screen.size)
|
||||
defer.returnValue((0, 0))
|
||||
|
||||
return self._with_client(_do)
|
||||
|
||||
@property
|
||||
def cursor_position(self) -> Tuple[int, int]:
|
||||
@@ -365,14 +477,8 @@ class VNCAutomationHandler(BaseAutomationHandler):
|
||||
try:
|
||||
btn = _BUTTON_MAP.get(button, 1)
|
||||
cx, cy = self._conn.cursor_position
|
||||
|
||||
def _drag():
|
||||
self._conn.mouse_down(cx, cy, btn)
|
||||
step = max(1, int(1 / max(duration, 0.01) * 5))
|
||||
self._conn.mouse_drag(x, y, step=step)
|
||||
self._conn.mouse_up(x, y, btn)
|
||||
|
||||
await asyncio.to_thread(_drag)
|
||||
step = max(1, int(1 / max(duration, 0.01) * 5))
|
||||
await asyncio.to_thread(self._conn.drag_to, cx, cy, x, y, btn, step)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
@@ -384,14 +490,7 @@ class VNCAutomationHandler(BaseAutomationHandler):
|
||||
if not path:
|
||||
return {"success": False, "error": "Empty path"}
|
||||
btn = _BUTTON_MAP.get(button, 1)
|
||||
|
||||
def _drag_path():
|
||||
self._conn.mouse_down(path[0][0], path[0][1], btn)
|
||||
for px, py in path[1:]:
|
||||
self._conn.mouse_move(px, py)
|
||||
self._conn.mouse_up(path[-1][0], path[-1][1], btn)
|
||||
|
||||
await asyncio.to_thread(_drag_path)
|
||||
await asyncio.to_thread(self._conn.drag_path, path, btn)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
@@ -443,10 +542,10 @@ class VNCAutomationHandler(BaseAutomationHandler):
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
async def scroll_down(self, clicks: int = 1) -> Dict[str, Any]:
|
||||
return await self.scroll(0, clicks)
|
||||
return await self.scroll(0, -clicks)
|
||||
|
||||
async def scroll_up(self, clicks: int = 1) -> Dict[str, Any]:
|
||||
return await self.scroll(0, -clicks)
|
||||
return await self.scroll(0, clicks)
|
||||
|
||||
# -- Screen info --------------------------------------------------------
|
||||
|
||||
|
||||
Reference in New Issue
Block a user