added simple guide for customizing computeragent

libs/python/agent/agent/agent.py

@@ -31,7 +31,8 @@ from .callbacks import (
     TrajectorySaverCallback, 
     BudgetManagerCallback,
     TelemetryCallback,
-    OperatorNormalizerCallback
+    OperatorNormalizerCallback,
+    PromptInstructionsCallback,
 )
 from .computers import (
     AsyncComputerHandler,
@@ -162,6 +163,7 @@ class ComputerAgent:
         custom_loop: Optional[Callable] = None,
         only_n_most_recent_images: Optional[int] = None,
         callbacks: Optional[List[Any]] = None,
+        instructions: Optional[str] = None,
         verbosity: Optional[int] = None,
         trajectory_dir: Optional[str | Path | dict] = None,
         max_retries: Optional[int] = 3,
@@ -180,6 +182,7 @@ class ComputerAgent:
             custom_loop: Custom agent loop function to use instead of auto-selection
             only_n_most_recent_images: If set, only keep the N most recent images in message history. Adds ImageRetentionCallback automatically.
             callbacks: List of AsyncCallbackHandler instances for preprocessing/postprocessing
+            instructions: Optional system instructions to be passed to the model
             verbosity: Logging level (logging.DEBUG, logging.INFO, etc.). If set, adds LoggingCallback automatically
             trajectory_dir: If set, saves trajectory data (screenshots, responses) to this directory. Adds TrajectorySaverCallback automatically.
             max_retries: Maximum number of retries for failed API calls
@@ -198,6 +201,7 @@ class ComputerAgent:
         self.custom_loop = custom_loop
         self.only_n_most_recent_images = only_n_most_recent_images
         self.callbacks = callbacks or []
+        self.instructions = instructions
         self.verbosity = verbosity
         self.trajectory_dir = trajectory_dir
         self.max_retries = max_retries
@@ -211,6 +215,10 @@ class ComputerAgent:
         # Prepend operator normalizer callback
         self.callbacks.insert(0, OperatorNormalizerCallback())
 
+        # Add prompt instructions callback if provided
+        if self.instructions:
+            self.callbacks.append(PromptInstructionsCallback(self.instructions))
+
         # Add telemetry callback if telemetry_enabled is set
         if self.telemetry_enabled:
             if isinstance(self.telemetry_enabled, bool):

libs/python/agent/agent/callbacks/__init__.py

@@ -9,6 +9,7 @@ from .trajectory_saver import TrajectorySaverCallback
 from .budget_manager import BudgetManagerCallback
 from .telemetry import TelemetryCallback
 from .operator_validator import OperatorNormalizerCallback
+from .prompt_instructions import PromptInstructionsCallback
 
 __all__ = [
     "AsyncCallbackHandler",
@@ -18,4 +19,5 @@ __all__ = [
     "BudgetManagerCallback",
     "TelemetryCallback",
     "OperatorNormalizerCallback",
+    "PromptInstructionsCallback",
 ]

libs/python/agent/agent/integrations/hud/__init__.py

@@ -20,6 +20,7 @@ from hud import trace
 
 from agent.agent import ComputerAgent as BaseComputerAgent
 from .proxy import FakeAsyncOpenAI
+from agent.callbacks import PromptInstructionsCallback
 
 
 # ---------------------------------------------------------------------------
@@ -47,6 +48,7 @@ class ProxyOperatorAgent(OperatorAgent):
         custom_loop: Any | None = None,
         only_n_most_recent_images: int | None = None,
         callbacks: list[Any] | None = None,
+        instructions: str | None = None,
         verbosity: int | None = None,
         max_retries: int | None = 3,
         screenshot_delay: float | int = 0.5,
@@ -68,12 +70,17 @@ class ProxyOperatorAgent(OperatorAgent):
         if tools:
             agent_tools.extend(tools)
 
+        # Build callbacks, injecting prompt instructions if provided
+        agent_callbacks = list(callbacks or [])
+        if instructions:
+            agent_callbacks.append(PromptInstructionsCallback(instructions))
+
         computer_agent = BaseComputerAgent(
             model=model,
             tools=agent_tools,
             custom_loop=custom_loop,
             only_n_most_recent_images=only_n_most_recent_images,
-            callbacks=callbacks,
+            callbacks=agent_callbacks,
             verbosity=verbosity,
             trajectory_dir=trajectory_dir,
             max_retries=max_retries,
@@ -96,7 +103,6 @@ class ProxyOperatorAgent(OperatorAgent):
 # Single-task runner
 # ---------------------------------------------------------------------------
 
-
 async def run_single_task(
     dataset: str | Dataset | list[dict[str, Any]],
     *,
@@ -108,6 +114,7 @@ async def run_single_task(
     custom_loop: Any | None = None,
     only_n_most_recent_images: int | None = None,
     callbacks: list[Any] | None = None,
+    instructions: str | None = None,
     verbosity: int | None = None,
     trajectory_dir: str | dict | None = None,
     max_retries: int | None = 3,
@@ -140,6 +147,7 @@ async def run_single_task(
             custom_loop=custom_loop,
             only_n_most_recent_images=only_n_most_recent_images,
             callbacks=callbacks,
+            instructions=instructions,
             verbosity=verbosity,
             trajectory_dir=trajectory_dir,
             max_retries=max_retries,
@@ -157,7 +165,6 @@ async def run_single_task(
 # Full-dataset runner
 # ---------------------------------------------------------------------------
 
-
 async def run_full_dataset(
     dataset: str | Dataset | list[dict[str, Any]],
     *,
@@ -173,6 +180,7 @@ async def run_full_dataset(
     custom_loop: Any | None = None,
     only_n_most_recent_images: int | None = 5,
     callbacks: list[Any] | None = None,
+    instructions: str | None = None,
     verbosity: int | None = None,
     max_retries: int | None = 3,
     screenshot_delay: float | int = 0.5,
@@ -207,6 +215,7 @@ async def run_full_dataset(
             "custom_loop": custom_loop,
             "only_n_most_recent_images": only_n_most_recent_images,
             "callbacks": callbacks,
+            "instructions": instructions,
             "verbosity": verbosity,
             "max_retries": max_retries,
             "screenshot_delay": screenshot_delay,