Fix issues with connecting to cloud vms, update tests

This commit is contained in:
Morgan Dean
2025-06-20 12:18:52 -07:00
parent 8be7dafa3e
commit 3e6ef65245
3 changed files with 270 additions and 229 deletions

View File

@@ -23,8 +23,7 @@ export class CloudComputer extends BaseComputer {
}
get ip() {
return "192.168.64.9";
//return `${this.name}.containers.cloud.trycua.com`;
return `${this.name}.containers.cloud.trycua.com`;
}
/**
@@ -36,8 +35,6 @@ export class CloudComputer extends BaseComputer {
return;
}
logger.info("Starting cloud computer...");
try {
// For cloud provider, the VM is already running, we just need to connect
const ipAddress = this.ip;
@@ -67,7 +64,7 @@ export class CloudComputer extends BaseComputer {
* Stop the cloud computer (disconnect interface)
*/
async stop(): Promise<void> {
logger.info("Stopping cloud computer...");
logger.info("Disconnecting from cloud computer...");
if (this.iface) {
this.iface.disconnect();
@@ -75,7 +72,7 @@ export class CloudComputer extends BaseComputer {
}
this.initialized = false;
logger.info("Cloud computer stopped");
logger.info("Disconnected from cloud computer");
}
/**

View File

@@ -39,7 +39,6 @@ export abstract class BaseComputerInterface {
protected ws: WebSocket;
protected apiKey?: string;
protected vmName?: string;
protected secure?: boolean;
protected logger = pino({ name: "interface-base" });
@@ -48,15 +47,13 @@ export abstract class BaseComputerInterface {
username = "lume",
password = "lume",
apiKey?: string,
vmName?: string,
secure?: boolean
vmName?: string
) {
this.ipAddress = ipAddress;
this.username = username;
this.password = password;
this.apiKey = apiKey;
this.vmName = vmName;
this.secure = secure;
// Initialize WebSocket with headers if needed
const headers: { [key: string]: string } = {};
@@ -74,7 +71,7 @@ export abstract class BaseComputerInterface {
* Subclasses can override this to customize the URI.
*/
protected get wsUri(): string {
const protocol = this.secure ? "wss" : "ws";
const protocol = this.apiKey ? "wss" : "ws";
// Check if ipAddress already includes a port
if (this.ipAddress.includes(":")) {
@@ -82,7 +79,7 @@ export abstract class BaseComputerInterface {
}
// Otherwise, append the default port
const port = this.secure ? "8443" : "8000";
const port = this.apiKey ? "8443" : "8000";
return `${protocol}://${this.ipAddress}:${port}/ws`;
}
@@ -99,6 +96,7 @@ export abstract class BaseComputerInterface {
await this.connect();
return;
} catch (error) {
console.log(error);
// Wait a bit before retrying
this.logger.error(
`Error connecting to websocket: ${JSON.stringify(error)}`
@@ -115,6 +113,42 @@ export abstract class BaseComputerInterface {
*/
public async connect(): Promise<void> {
if (this.ws.readyState === WebSocket.OPEN) {
// send authentication message if needed
if (this.apiKey && this.vmName) {
this.logger.info("Performing authentication handshake...");
const authMessage = {
command: "authenticate",
params: {
api_key: this.apiKey,
container_name: this.vmName,
},
};
return new Promise<void>((resolve, reject) => {
const authHandler = (data: WebSocket.RawData) => {
try {
const authResult = JSON.parse(data.toString());
if (!authResult.success) {
const errorMsg = authResult.error || "Authentication failed";
this.logger.error(`Authentication failed: ${errorMsg}`);
this.ws.close();
reject(new Error(`Authentication failed: ${errorMsg}`));
} else {
this.logger.info("Authentication successful");
this.ws.off("message", authHandler);
resolve();
}
} catch (error) {
this.ws.off("message", authHandler);
reject(error);
}
};
this.ws.on("message", authHandler);
this.ws.send(JSON.stringify(authMessage));
});
}
return;
}

View File

@@ -8,7 +8,7 @@ describe("MacOSComputerInterface", () => {
ipAddress: "localhost",
username: "testuser",
password: "testpass",
apiKey: "test-api-key",
// apiKey: "test-api-key", No API Key for local testing
vmName: "test-vm",
};
@@ -37,18 +37,9 @@ describe("MacOSComputerInterface", () => {
testParams.ipAddress = `localhost:${serverPort}`;
// Handle WebSocket connections
wss.on("connection", (ws, req) => {
wss.on("connection", (ws) => {
connectedClients.push(ws);
// Verify authentication headers
const apiKey = req.headers["x-api-key"];
const vmName = req.headers["x-vm-name"];
if (apiKey !== testParams.apiKey || vmName !== testParams.vmName) {
ws.close(1008, "Unauthorized");
return;
}
// Handle incoming messages
ws.on("message", (data) => {
try {
@@ -58,81 +49,107 @@ describe("MacOSComputerInterface", () => {
// Send appropriate responses based on action
switch (message.command) {
case "screenshot":
ws.send(JSON.stringify({
image_data: Buffer.from("fake-screenshot-data").toString("base64"),
success: true
}));
ws.send(
JSON.stringify({
image_data: Buffer.from("fake-screenshot-data").toString(
"base64"
),
success: true,
})
);
break;
case "get_screen_size":
ws.send(JSON.stringify({
size: { width: 1920, height: 1080 },
success: true
}));
ws.send(
JSON.stringify({
size: { width: 1920, height: 1080 },
success: true,
})
);
break;
case "get_cursor_position":
ws.send(JSON.stringify({
position: { x: 100, y: 200 },
success: true
}));
ws.send(
JSON.stringify({
position: { x: 100, y: 200 },
success: true,
})
);
break;
case "copy_to_clipboard":
ws.send(JSON.stringify({
content: "clipboard content",
success: true
}));
ws.send(
JSON.stringify({
content: "clipboard content",
success: true,
})
);
break;
case "file_exists":
ws.send(JSON.stringify({
exists: true,
success: true
}));
ws.send(
JSON.stringify({
exists: true,
success: true,
})
);
break;
case "directory_exists":
ws.send(JSON.stringify({
exists: true,
success: true
}));
ws.send(
JSON.stringify({
exists: true,
success: true,
})
);
break;
case "list_dir":
ws.send(JSON.stringify({
files: ["file1.txt", "file2.txt"],
success: true
}));
ws.send(
JSON.stringify({
files: ["file1.txt", "file2.txt"],
success: true,
})
);
break;
case "read_text":
ws.send(JSON.stringify({
content: "file content",
success: true
}));
ws.send(
JSON.stringify({
content: "file content",
success: true,
})
);
break;
case "read_bytes":
ws.send(JSON.stringify({
content_b64: Buffer.from("binary content").toString("base64"),
success: true
}));
ws.send(
JSON.stringify({
content_b64: Buffer.from("binary content").toString("base64"),
success: true,
})
);
break;
case "run_command":
ws.send(JSON.stringify({
stdout: "command output",
stderr: "",
success: true
}));
ws.send(
JSON.stringify({
stdout: "command output",
stderr: "",
success: true,
})
);
break;
case "get_accessibility_tree":
ws.send(JSON.stringify({
role: "window",
title: "Test Window",
bounds: { x: 0, y: 0, width: 1920, height: 1080 },
children: [],
success: true
}));
ws.send(
JSON.stringify({
role: "window",
title: "Test Window",
bounds: { x: 0, y: 0, width: 1920, height: 1080 },
children: [],
success: true,
})
);
break;
case "to_screen_coordinates":
case "to_screenshot_coordinates":
ws.send(JSON.stringify({
coordinates: [message.params?.x || 0, message.params?.y || 0],
success: true
}));
ws.send(
JSON.stringify({
coordinates: [message.params?.x || 0, message.params?.y || 0],
success: true,
})
);
break;
default:
// For all other actions, just send success
@@ -171,9 +188,8 @@ describe("MacOSComputerInterface", () => {
testParams.ipAddress,
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
await macosInterface.connect();
@@ -201,8 +217,7 @@ describe("MacOSComputerInterface", () => {
testParams.username,
testParams.password,
undefined,
undefined,
false
undefined
);
await macosInterface.connect();
@@ -223,9 +238,8 @@ describe("MacOSComputerInterface", () => {
testParams.ipAddress,
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
await macosInterface.connect();
});
@@ -238,87 +252,87 @@ describe("MacOSComputerInterface", () => {
it("should send mouse_down command", async () => {
await macosInterface.mouseDown(100, 200, "left");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "mouse_down",
params: {
x: 100,
y: 200,
button: "left"
}
button: "left",
},
});
});
it("should send mouse_up command", async () => {
await macosInterface.mouseUp(100, 200, "right");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "mouse_up",
params: {
x: 100,
y: 200,
button: "right"
}
button: "right",
},
});
});
it("should send left_click command", async () => {
await macosInterface.leftClick(150, 250);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "left_click",
params: {
x: 150,
y: 250
}
y: 250,
},
});
});
it("should send right_click command", async () => {
await macosInterface.rightClick(200, 300);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "right_click",
params: {
x: 200,
y: 300
}
y: 300,
},
});
});
it("should send double_click command", async () => {
await macosInterface.doubleClick(250, 350);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "double_click",
params: {
x: 250,
y: 350
}
y: 350,
},
});
});
it("should send move_cursor command", async () => {
await macosInterface.moveCursor(300, 400);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "move_cursor",
params: {
x: 300,
y: 400
}
y: 400,
},
});
});
it("should send drag_to command", async () => {
await macosInterface.dragTo(400, 500, "left", 1.5);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "drag_to",
@@ -326,23 +340,27 @@ describe("MacOSComputerInterface", () => {
x: 400,
y: 500,
button: "left",
duration: 1.5
}
duration: 1.5,
},
});
});
it("should send drag command with path", async () => {
const path: Array<[number, number]> = [[100, 100], [200, 200], [300, 300]];
const path: Array<[number, number]> = [
[100, 100],
[200, 200],
[300, 300],
];
await macosInterface.drag(path, "middle", 2.0);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "drag",
params: {
path: path,
button: "middle",
duration: 2.0
}
duration: 2.0,
},
});
});
});
@@ -355,9 +373,8 @@ describe("MacOSComputerInterface", () => {
testParams.ipAddress,
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
await macosInterface.connect();
});
@@ -370,61 +387,61 @@ describe("MacOSComputerInterface", () => {
it("should send key_down command", async () => {
await macosInterface.keyDown("a");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "key_down",
params: {
key: "a"
}
key: "a",
},
});
});
it("should send key_up command", async () => {
await macosInterface.keyUp("b");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "key_up",
params: {
key: "b"
}
key: "b",
},
});
});
it("should send type_text command", async () => {
await macosInterface.typeText("Hello, World!");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "type_text",
params: {
text: "Hello, World!"
}
text: "Hello, World!",
},
});
});
it("should send press_key command", async () => {
await macosInterface.pressKey("enter");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "press_key",
params: {
key: "enter"
}
key: "enter",
},
});
});
it("should send hotkey command", async () => {
await macosInterface.hotkey("cmd", "c");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "hotkey",
params: {
keys: ["cmd", "c"]
}
keys: ["cmd", "c"],
},
});
});
});
@@ -437,9 +454,8 @@ describe("MacOSComputerInterface", () => {
testParams.ipAddress,
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
await macosInterface.connect();
});
@@ -452,38 +468,38 @@ describe("MacOSComputerInterface", () => {
it("should send scroll command", async () => {
await macosInterface.scroll(10, -5);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "scroll",
params: {
x: 10,
y: -5
}
y: -5,
},
});
});
it("should send scroll_down command", async () => {
await macosInterface.scrollDown(3);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "scroll_down",
params: {
clicks: 3
}
clicks: 3,
},
});
});
it("should send scroll_up command", async () => {
await macosInterface.scrollUp(2);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "scroll_up",
params: {
clicks: 2
}
clicks: 2,
},
});
});
});
@@ -496,9 +512,8 @@ describe("MacOSComputerInterface", () => {
testParams.ipAddress,
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
await macosInterface.connect();
});
@@ -511,38 +526,38 @@ describe("MacOSComputerInterface", () => {
it("should get screenshot", async () => {
const screenshot = await macosInterface.screenshot();
expect(screenshot).toBeInstanceOf(Buffer);
expect(screenshot.toString()).toBe("fake-screenshot-data");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "screenshot",
params: {}
params: {},
});
});
it("should get screen size", async () => {
const size = await macosInterface.getScreenSize();
expect(size).toEqual({ width: 1920, height: 1080 });
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "get_screen_size",
params: {}
params: {},
});
});
it("should get cursor position", async () => {
const position = await macosInterface.getCursorPosition();
expect(position).toEqual({ x: 100, y: 200 });
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "get_cursor_position",
params: {}
params: {},
});
});
});
@@ -555,9 +570,8 @@ describe("MacOSComputerInterface", () => {
testParams.ipAddress,
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
await macosInterface.connect();
});
@@ -570,25 +584,25 @@ describe("MacOSComputerInterface", () => {
it("should copy to clipboard", async () => {
const text = await macosInterface.copyToClipboard();
expect(text).toBe("clipboard content");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "copy_to_clipboard",
params: {}
params: {},
});
});
it("should set clipboard", async () => {
await macosInterface.setClipboard("new clipboard text");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "set_clipboard",
params: {
text: "new clipboard text"
}
text: "new clipboard text",
},
});
});
});
@@ -601,9 +615,8 @@ describe("MacOSComputerInterface", () => {
testParams.ipAddress,
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
await macosInterface.connect();
});
@@ -616,150 +629,150 @@ describe("MacOSComputerInterface", () => {
it("should check file exists", async () => {
const exists = await macosInterface.fileExists("/path/to/file");
expect(exists).toBe(true);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "file_exists",
params: {
path: "/path/to/file"
}
path: "/path/to/file",
},
});
});
it("should check directory exists", async () => {
const exists = await macosInterface.directoryExists("/path/to/dir");
expect(exists).toBe(true);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "directory_exists",
params: {
path: "/path/to/dir"
}
path: "/path/to/dir",
},
});
});
it("should list directory", async () => {
const files = await macosInterface.listDir("/path/to/dir");
expect(files).toEqual(["file1.txt", "file2.txt"]);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "list_dir",
params: {
path: "/path/to/dir"
}
path: "/path/to/dir",
},
});
});
it("should read text file", async () => {
const content = await macosInterface.readText("/path/to/file.txt");
expect(content).toBe("file content");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "read_text",
params: {
path: "/path/to/file.txt"
}
path: "/path/to/file.txt",
},
});
});
it("should write text file", async () => {
await macosInterface.writeText("/path/to/file.txt", "new content");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "write_text",
params: {
path: "/path/to/file.txt",
content: "new content"
}
content: "new content",
},
});
});
it("should read binary file", async () => {
const content = await macosInterface.readBytes("/path/to/file.bin");
expect(content).toBeInstanceOf(Buffer);
expect(content.toString()).toBe("binary content");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "read_bytes",
params: {
path: "/path/to/file.bin"
}
path: "/path/to/file.bin",
},
});
});
it("should write binary file", async () => {
const buffer = Buffer.from("binary data");
await macosInterface.writeBytes("/path/to/file.bin", buffer);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "write_bytes",
params: {
path: "/path/to/file.bin",
content_b64: buffer.toString("base64")
}
content_b64: buffer.toString("base64"),
},
});
});
it("should delete file", async () => {
await macosInterface.deleteFile("/path/to/file");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "delete_file",
params: {
path: "/path/to/file"
}
path: "/path/to/file",
},
});
});
it("should create directory", async () => {
await macosInterface.createDir("/path/to/new/dir");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "create_dir",
params: {
path: "/path/to/new/dir"
}
path: "/path/to/new/dir",
},
});
});
it("should delete directory", async () => {
await macosInterface.deleteDir("/path/to/dir");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "delete_dir",
params: {
path: "/path/to/dir"
}
path: "/path/to/dir",
},
});
});
it("should run command", async () => {
const [stdout, stderr] = await macosInterface.runCommand("ls -la");
expect(stdout).toBe("command output");
expect(stderr).toBe("");
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "run_command",
params: {
command: "ls -la"
}
command: "ls -la",
},
});
});
});
@@ -772,9 +785,8 @@ describe("MacOSComputerInterface", () => {
testParams.ipAddress,
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
await macosInterface.connect();
});
@@ -787,51 +799,51 @@ describe("MacOSComputerInterface", () => {
it("should get accessibility tree", async () => {
const tree = await macosInterface.getAccessibilityTree();
expect(tree).toEqual({
role: "window",
title: "Test Window",
bounds: { x: 0, y: 0, width: 1920, height: 1080 },
children: [],
success: true
success: true,
});
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "get_accessibility_tree",
params: {}
params: {},
});
});
it("should convert to screen coordinates", async () => {
const [x, y] = await macosInterface.toScreenCoordinates(100, 200);
expect(x).toBe(100);
expect(y).toBe(200);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "to_screen_coordinates",
params: {
x: 100,
y: 200
}
y: 200,
},
});
});
it("should convert to screenshot coordinates", async () => {
const [x, y] = await macosInterface.toScreenshotCoordinates(300, 400);
expect(x).toBe(300);
expect(y).toBe(400);
const lastMessage = receivedMessages[receivedMessages.length - 1];
expect(lastMessage).toEqual({
command: "to_screenshot_coordinates",
params: {
x: 300,
y: 400
}
y: 400,
},
});
});
});
@@ -843,9 +855,8 @@ describe("MacOSComputerInterface", () => {
"localhost:9999",
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
// Connection should fail
@@ -867,15 +878,16 @@ describe("MacOSComputerInterface", () => {
`localhost:${errorPort}`,
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
await macosInterface.connect();
// Command should throw error
await expect(macosInterface.leftClick(100, 100)).rejects.toThrow("Command failed");
await expect(macosInterface.leftClick(100, 100)).rejects.toThrow(
"Command failed"
);
await macosInterface.disconnect();
await new Promise<void>((resolve) => {
@@ -888,9 +900,8 @@ describe("MacOSComputerInterface", () => {
testParams.ipAddress,
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
await macosInterface.connect();
@@ -912,9 +923,8 @@ describe("MacOSComputerInterface", () => {
testParams.ipAddress,
testParams.username,
testParams.password,
testParams.apiKey,
testParams.vmName,
false
undefined,
testParams.vmName
);
await macosInterface.connect();