From fb174d6aa4e5c74db74616de6342d24eda1e2956 Mon Sep 17 00:00:00 2001 From: Dillon DuPont Date: Fri, 24 Oct 2025 16:27:58 -0700 Subject: [PATCH] Add get_desktop_environment, set_wallpaper --- docs/content/docs/computer-sdk/commands.mdx | 64 ++++++++++++------- .../computer_server/handlers/base.py | 20 ++++++ .../computer_server/handlers/factory.py | 20 +++++- .../computer_server/handlers/generic.py | 44 ++++++++++++- .../computer-server/computer_server/main.py | 5 +- .../computer/computer/interface/base.py | 20 ++++++ .../computer/computer/interface/generic.py | 12 ++++ .../typescript/computer/src/interface/base.ts | 4 ++ .../computer/src/interface/macos.ts | 25 +++++++- 9 files changed, 186 insertions(+), 28 deletions(-) diff --git a/docs/content/docs/computer-sdk/commands.mdx b/docs/content/docs/computer-sdk/commands.mdx index d8e80493..4d36baa4 100644 --- a/docs/content/docs/computer-sdk/commands.mdx +++ b/docs/content/docs/computer-sdk/commands.mdx @@ -13,16 +13,12 @@ Execute shell commands and get detailed results: - ```python - # Run shell command result = await - computer.interface.run_command(cmd) # result.stdout, result.stderr, result.returncode - ``` + ```python # Run shell command result = await computer.interface.run_command(cmd) # + result.stdout, result.stderr, result.returncode ``` - ```typescript - // Run shell command const result = await - computer.interface.runCommand(cmd); // result.stdout, result.stderr, result.returncode - ``` + ```typescript // Run shell command const result = await computer.interface.runCommand(cmd); // + result.stdout, result.stderr, result.returncode ``` @@ -108,20 +104,14 @@ Mouse wheel and scrolling control: - ```python - # Scrolling - await computer.interface.scroll(x, y) # Scroll the mouse wheel - await computer.interface.scroll_down(clicks) # Scroll down await - computer.interface.scroll_up(clicks) # Scroll up - ``` + ```python # Scrolling await computer.interface.scroll(x, y) # Scroll the mouse wheel await + computer.interface.scroll_down(clicks) # Scroll down await computer.interface.scroll_up(clicks) + # Scroll up ``` - ```typescript - // Scrolling - await computer.interface.scroll(x, y); // Scroll the mouse wheel - await computer.interface.scrollDown(clicks); // Scroll down - await computer.interface.scrollUp(clicks); // Scroll up - ``` + ```typescript // Scrolling await computer.interface.scroll(x, y); // Scroll the mouse wheel + await computer.interface.scrollDown(clicks); // Scroll down await + computer.interface.scrollUp(clicks); // Scroll up ``` @@ -149,6 +139,35 @@ Screen capture and display information: +## Desktop Actions + +Control desktop environment features like wallpaper: + + + + ```python + # Get current desktop environment (e.g., 'xfce4', 'gnome', 'kde', 'mac', 'windows') + env = await computer.interface.get_desktop_environment() + print(env) # "xfce4" + + # Set desktop wallpaper to an image file accessible on the VM + await computer.interface.set_wallpaper("/home/cua/shared/wallpaper.png") + ``` + + + + ```typescript + // Get current desktop environment + const env = await computer.interface.getDesktopEnvironment(); + print(env) # "xfce4" + + // Set desktop wallpaper to an image file accessible on the VM + await computer.interface.setWallpaper('/home/cua/shared/wallpaper.png'); + ``` + + + + ## Clipboard Actions System clipboard management: @@ -240,7 +259,7 @@ Access system accessibility information: // Get accessibility tree await computer.interface.getAccessibilityTree(); -``` +```` @@ -277,4 +296,5 @@ Manage Python environments: ``` - \ No newline at end of file + +```` diff --git a/libs/python/computer-server/computer_server/handlers/base.py b/libs/python/computer-server/computer_server/handlers/base.py index 42b27a0c..73250d44 100644 --- a/libs/python/computer-server/computer_server/handlers/base.py +++ b/libs/python/computer-server/computer_server/handlers/base.py @@ -85,6 +85,26 @@ class BaseFileHandler(ABC): pass +class BaseDesktopHandler(ABC): + """Abstract base class for OS-specific desktop handlers. + + Categories: + - Wallpaper Actions: Methods for wallpaper operations + - Desktop shortcut actions: Methods for managing desktop shortcuts + """ + + # Wallpaper Actions + @abstractmethod + async def get_desktop_environment(self) -> Dict[str, Any]: + """Get the current desktop environment name.""" + pass + + @abstractmethod + async def set_wallpaper(self, path: str) -> Dict[str, Any]: + """Set the desktop wallpaper to the file at path.""" + pass + + class BaseAutomationHandler(ABC): """Abstract base class for OS-specific automation handlers. diff --git a/libs/python/computer-server/computer_server/handlers/factory.py b/libs/python/computer-server/computer_server/handlers/factory.py index 3c90935f..77d88e5f 100644 --- a/libs/python/computer-server/computer_server/handlers/factory.py +++ b/libs/python/computer-server/computer_server/handlers/factory.py @@ -4,7 +4,12 @@ from typing import Tuple, Type from computer_server.diorama.base import BaseDioramaHandler -from .base import BaseAccessibilityHandler, BaseAutomationHandler, BaseFileHandler +from .base import ( + BaseAccessibilityHandler, + BaseAutomationHandler, + BaseDesktopHandler, + BaseFileHandler, +) # Conditionally import platform-specific handlers system = platform.system().lower() @@ -17,7 +22,7 @@ elif system == "linux": elif system == "windows": from .windows import WindowsAccessibilityHandler, WindowsAutomationHandler -from .generic import GenericFileHandler +from .generic import GenericDesktopHandler, GenericFileHandler class HandlerFactory: @@ -50,7 +55,13 @@ class HandlerFactory: @staticmethod def create_handlers() -> ( - Tuple[BaseAccessibilityHandler, BaseAutomationHandler, BaseDioramaHandler, BaseFileHandler] + Tuple[ + BaseAccessibilityHandler, + BaseAutomationHandler, + BaseDioramaHandler, + BaseFileHandler, + BaseDesktopHandler, + ] ): """Create and return appropriate handlers for the current OS. @@ -70,6 +81,7 @@ class HandlerFactory: MacOSAutomationHandler(), MacOSDioramaHandler(), GenericFileHandler(), + GenericDesktopHandler(), ) elif os_type == "linux": return ( @@ -77,6 +89,7 @@ class HandlerFactory: LinuxAutomationHandler(), BaseDioramaHandler(), GenericFileHandler(), + GenericDesktopHandler(), ) elif os_type == "windows": return ( @@ -84,6 +97,7 @@ class HandlerFactory: WindowsAutomationHandler(), BaseDioramaHandler(), GenericFileHandler(), + GenericDesktopHandler(), ) else: raise NotImplementedError(f"OS '{os_type}' is not supported") diff --git a/libs/python/computer-server/computer_server/handlers/generic.py b/libs/python/computer-server/computer_server/handlers/generic.py index da29f1c3..c7348312 100644 --- a/libs/python/computer-server/computer_server/handlers/generic.py +++ b/libs/python/computer-server/computer_server/handlers/generic.py @@ -2,6 +2,7 @@ Generic handlers for all OSes. Includes: +- DesktopHandler - FileHandler """ @@ -10,7 +11,8 @@ import base64 from pathlib import Path from typing import Any, Dict, Optional -from .base import BaseFileHandler +from ..utils import wallpaper +from .base import BaseDesktopHandler, BaseFileHandler def resolve_path(path: str) -> Path: @@ -25,6 +27,46 @@ def resolve_path(path: str) -> Path: return Path(path).expanduser().resolve() +class GenericDesktopHandler(BaseDesktopHandler): + """ + Generic desktop handler providing desktop-related operations. + + Implements: + - get_desktop_environment: detect current desktop environment + - set_wallpaper: set desktop wallpaper path + """ + + async def get_desktop_environment(self) -> Dict[str, Any]: + """ + Get the current desktop environment. + + Returns: + Dict containing 'success' boolean and either 'environment' string or 'error' string + """ + try: + env = wallpaper.get_desktop_environment() + return {"success": True, "environment": env} + except Exception as e: + return {"success": False, "error": str(e)} + + async def set_wallpaper(self, path: str) -> Dict[str, Any]: + """ + Set the desktop wallpaper to the specified path. + + Args: + path: The file path to set as wallpaper + + Returns: + Dict containing 'success' boolean and optionally 'error' string + """ + try: + file_path = resolve_path(path) + ok = wallpaper.set_wallpaper(str(file_path)) + return {"success": bool(ok)} + except Exception as e: + return {"success": False, "error": str(e)} + + class GenericFileHandler(BaseFileHandler): """ Generic file handler that provides file system operations for all operating systems. diff --git a/libs/python/computer-server/computer_server/main.py b/libs/python/computer-server/computer_server/main.py index 9c0c052b..005c7ddb 100644 --- a/libs/python/computer-server/computer_server/main.py +++ b/libs/python/computer-server/computer_server/main.py @@ -75,7 +75,7 @@ except Exception: except Exception: package_version = "unknown" -accessibility_handler, automation_handler, diorama_handler, file_handler = ( +accessibility_handler, automation_handler, diorama_handler, file_handler, desktop_handler = ( HandlerFactory.create_handlers() ) handlers = { @@ -99,6 +99,9 @@ handlers = { "delete_file": file_handler.delete_file, "create_dir": file_handler.create_dir, "delete_dir": file_handler.delete_dir, + # Desktop commands + "get_desktop_environment": desktop_handler.get_desktop_environment, + "set_wallpaper": desktop_handler.set_wallpaper, # Mouse commands "mouse_down": automation_handler.mouse_down, "mouse_up": automation_handler.mouse_up, diff --git a/libs/python/computer/computer/interface/base.py b/libs/python/computer/computer/interface/base.py index 8fa40025..42c7ab0f 100644 --- a/libs/python/computer/computer/interface/base.py +++ b/libs/python/computer/computer/interface/base.py @@ -436,6 +436,26 @@ class BaseComputerInterface(ABC): """ pass + # Desktop actions + @abstractmethod + async def get_desktop_environment(self) -> str: + """Get the current desktop environment. + + Returns: + The name of the current desktop environment. + """ + pass + + @abstractmethod + async def set_wallpaper(self, path: str) -> None: + """Set the desktop wallpaper to the specified path. + + Args: + path: The file path to set as wallpaper + """ + pass + + # Shell actions @abstractmethod async def run_command(self, command: str) -> CommandResult: """Run shell command and return structured result. diff --git a/libs/python/computer/computer/interface/generic.py b/libs/python/computer/computer/interface/generic.py index 7cd2372f..7cf47461 100644 --- a/libs/python/computer/computer/interface/generic.py +++ b/libs/python/computer/computer/interface/generic.py @@ -487,6 +487,18 @@ class GenericComputerInterface(BaseComputerInterface): raise RuntimeError(result.get("error", "Failed to list directory")) return result.get("files", []) + # Desktop actions + async def get_desktop_environment(self) -> str: + result = await self._send_command("get_desktop_environment") + if not result.get("success", False): + raise RuntimeError(result.get("error", "Failed to get desktop environment")) + return result.get("environment", "unknown") + + async def set_wallpaper(self, path: str) -> None: + result = await self._send_command("set_wallpaper", {"path": path}) + if not result.get("success", False): + raise RuntimeError(result.get("error", "Failed to set wallpaper")) + # Command execution async def run_command(self, command: str) -> CommandResult: result = await self._send_command("run_command", {"command": command}) diff --git a/libs/typescript/computer/src/interface/base.ts b/libs/typescript/computer/src/interface/base.ts index 4cd85f18..e41c4416 100644 --- a/libs/typescript/computer/src/interface/base.ts +++ b/libs/typescript/computer/src/interface/base.ts @@ -314,6 +314,10 @@ export abstract class BaseComputerInterface { abstract getScreenSize(): Promise; abstract getCursorPosition(): Promise; + // Desktop Actions + abstract getDesktopEnvironment(): Promise; + abstract setWallpaper(path: string): Promise; + // Clipboard Actions abstract copyToClipboard(): Promise; abstract setClipboard(text: string): Promise; diff --git a/libs/typescript/computer/src/interface/macos.ts b/libs/typescript/computer/src/interface/macos.ts index c1c05454..86522042 100644 --- a/libs/typescript/computer/src/interface/macos.ts +++ b/libs/typescript/computer/src/interface/macos.ts @@ -3,8 +3,8 @@ */ import type { ScreenSize } from '../types'; -import { BaseComputerInterface } from './base'; import type { AccessibilityNode, CursorPosition, MouseButton } from './base'; +import { BaseComputerInterface } from './base'; export class MacOSComputerInterface extends BaseComputerInterface { // Mouse Actions @@ -212,6 +212,29 @@ export class MacOSComputerInterface extends BaseComputerInterface { return response.position as CursorPosition; } + // Desktop Actions + /** + * Get the current desktop environment string (e.g., 'xfce4', 'gnome', 'kde', 'mac', 'windows'). + */ + async getDesktopEnvironment(): Promise { + const response = await this.sendCommand('get_desktop_environment'); + if (!response.success) { + throw new Error((response.error as string) || 'Failed to get desktop environment'); + } + return (response.environment as string) || 'unknown'; + } + + /** + * Set the desktop wallpaper image. + * @param path Absolute path to the image file on the VM + */ + async setWallpaper(path: string): Promise { + const response = await this.sendCommand('set_wallpaper', { path }); + if (!response.success) { + throw new Error((response.error as string) || 'Failed to set wallpaper'); + } + } + // Clipboard Actions /** * Copy current selection to clipboard and return the content.