diff --git a/notebooks/eval_osworld.ipynb b/notebooks/eval_osworld.ipynb index 0d58f58e..a287022c 100644 --- a/notebooks/eval_osworld.ipynb +++ b/notebooks/eval_osworld.ipynb @@ -50,18 +50,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/dillondupont/cua-clean/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "# Import the HUD-integrated ComputerAgent\n", "from agent.integrations.hud import ComputerAgent" @@ -93,7 +84,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -121,14 +112,14 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[INFO] 2025-08-08 15:16:46,133 | hud.environment | View the live trace at https://app.hud.so/trace/662fd59f-5a8d-4205-9b88-32c00d0feab0\n" + "[INFO] 2025-08-08 19:08:17,078 | hud.environment | View the live trace at https://app.hud.so/trace/ca88c178-cf40-499b-8ad3-d5d60348d9fe\n" ] }, { @@ -147,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -156,7 +147,7 @@ "\n", "
\n", "
\n", - " \n", "
\n", "
\n", @@ -172,10 +163,10 @@ { "data": { "text/plain": [ - "'\\n
\\n
\\n \\n
\\n
\\n '" + "'\\n
\\n
\\n \\n
\\n
\\n '" ] }, - "execution_count": 6, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -188,21 +179,21 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Test with Claude Model\n", + "## Test with any supported CUA model\n", "\n", - "The ComputerAgent can use Claude models just like the original ClaudeAgent:" + "The ComputerAgent integration can use Claude, OpenAI, UI-TARS, or composed models just like the original ComputerAgent:" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Created Claude agent: computeragent-claude-3-5-sonnet-20241022\n" + "Created agent: computeragent-computer-use-preview\n" ] } ], @@ -210,17 +201,20 @@ "import logging\n", "# Create ComputerAgent with Claude\n", "claude_agent = ComputerAgent(\n", - " model=\"anthropic/claude-3-5-sonnet-20241022\",\n", - " environment=\"linux\", # OSWorld typically uses Linux\n", + " # model=\"anthropic/claude-3-5-sonnet-20241022\",\n", + " model=\"openai/computer-use-preview\",\n", + " # environment=\"linux\", # OSWorld typically uses Linux\n", + " environment=\"browser\", # SheetBench uses the browser\n", + " trajectory_dir=\"trajectories\",\n", " verbosity=logging.INFO,\n", ")\n", "\n", - "print(f\"Created Claude agent: {claude_agent.name}\")" + "print(f\"Created agent: {claude_agent.name}\")" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -235,15 +229,127 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-08 15:17:04,030 - agent.ComputerAgent - INFO - LLM processing started with 1 messages\n" + "2025-08-08 19:14:10,479 - agent.ComputerAgent - INFO - LLM processing started with 1 messages\n", + "2025-08-08 19:14:18,867 - agent.ComputerAgent - INFO - Computer: click({'button': 'left', 'x': 55, 'y': 149})\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Agent's action: [ResponseAction(type='response', reasoning='I\\'ll help you complete this task step by step, but I notice that I don\\'t have any input data or access to Excel through the available functions. The only function I have access to is the \"computer\" function which allows for basic desktop interaction.\\n\\nTo properly assist you, I would need:\\n1. The actual input data you want to analyze\\n2. Access to Excel or another spreadsheet tool to perform the calculations\\n\\nCould you please provide the input data and confirm if there\\'s a specific way to access Excel or the data file on this system?\\n\\nOnce provided, I can help calculate correlations between volume and next day price changes, sort the data as specified, and format the results according to your requirements.', logs={'conversation_length': 2}, text='I\\'ll help you complete this task step by step, but I notice that I don\\'t have any input data or access to Excel through the available functions. The only function I have access to is the \"computer\" function which allows for basic desktop interaction.\\n\\nTo properly assist you, I would need:\\n1. The actual input data you want to analyze\\n2. Access to Excel or another spreadsheet tool to perform the calculations\\n\\nCould you please provide the input data and confirm if there\\'s a specific way to access Excel or the data file on this system?\\n\\nOnce provided, I can help calculate correlations between volume and next day price changes, sort the data as specified, and format the results according to your requirements.')]\n", - "Task completed after 1 steps\n" + "Agent's action: [ClickAction(type='click', reasoning='Sorting dataset for analysis preparation', logs={'conversation_length': 3}, point=Point(x=77, y=174), button='left', pattern=None, hold_keys=None)]\n", + "========= Step 2 ==========\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-08-08 19:14:24,566 - agent.ComputerAgent - INFO - LLM processing started with 4 messages\n", + "2025-08-08 19:14:30,430 - agent.ComputerAgent - INFO - Computer: keypress({'keys': ['CTRL', 'A']})\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Agent's action: [PressAction(type='press', reasoning='Sorting dataset for analysis preparation', logs={'conversation_length': 5}, keys=['ctrl', 'a'])]\n", + "========= Step 3 ==========\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-08-08 19:14:36,137 - agent.ComputerAgent - INFO - LLM processing started with 6 messages\n", + "2025-08-08 19:14:42,483 - agent.ComputerAgent - INFO - Computer: click({'button': 'left', 'x': 73, 'y': 151})\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Agent's action: [ClickAction(type='click', reasoning='Sorting dataset for analysis preparation', logs={'conversation_length': 7}, point=Point(x=102, y=176), button='left', pattern=None, hold_keys=None)]\n", + "========= Step 4 ==========\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-08-08 19:14:48,687 - agent.ComputerAgent - INFO - LLM processing started with 8 messages\n", + "2025-08-08 19:14:59,516 - agent.ComputerAgent - INFO - Computer: keypress({'keys': ['CTRL', 'A']})\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Agent's action: [PressAction(type='press', reasoning='Sorting dataset for analysis preparation', logs={'conversation_length': 9}, keys=['ctrl', 'a'])]\n", + "========= Step 5 ==========\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-08-08 19:15:05,229 - agent.ComputerAgent - INFO - LLM processing started with 10 messages\n", + "2025-08-08 19:15:15,153 - agent.ComputerAgent - INFO - Computer: drag({'path': [{'x': 55, 'y': 147}, {'x': 319, 'y': 713}]})\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Agent's action: [DragAction(type='drag', reasoning='Highlighting data for sorting preparation', logs={'conversation_length': 12}, path=[Point(x=77, y=172), Point(x=448, y=835)], pattern=None, hold_keys=None)]\n", + "========= Step 6 ==========\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-08-08 19:15:21,362 - agent.ComputerAgent - INFO - LLM processing started with 13 messages\n", + "2025-08-08 19:15:33,774 - agent.ComputerAgent - INFO - Computer: click({'button': 'left', 'x': 229, 'y': 41})\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Agent's action: [ClickAction(type='click', reasoning='Opening sort options for data', logs={'conversation_length': 15}, point=Point(x=322, y=48), button='left', pattern=None, hold_keys=None)]\n", + "========= Step 7 ==========\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-08-08 19:15:39,973 - agent.ComputerAgent - INFO - LLM processing started with 16 messages\n", + "2025-08-08 19:15:52,928 - agent.ComputerAgent - INFO - Computer: click({'button': 'left', 'x': 430, 'y': 96})\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Agent's action: [ClickAction(type='click', reasoning='Choosing \"Sort range\" for sorting', logs={'conversation_length': 18}, point=Point(x=604, y=112), button='left', pattern=None, hold_keys=None)]\n", + "========= Step 8 ==========\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-08-08 19:15:59,611 - agent.ComputerAgent - INFO - LLM processing started with 19 messages\n", + "2025-08-08 19:16:17,003 - agent.ComputerAgent - INFO - Computer: click({'button': 'left', 'x': 530, 'y': 172})\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Agent's action: [ClickAction(type='click', reasoning='Accessing advanced sorting options now', logs={'conversation_length': 21}, point=Point(x=745, y=201), button='left', pattern=None, hold_keys=None)]\n" ] } ], @@ -280,9 +386,353 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== Final Evaluation ===\n", + "{'error': None,\n", + " 'gold_file_url': 'https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/615426c8-9df7-4ffa-92e9-200134a84da9/gold_solution_2.xlsx?',\n", + " 'logs': 'INFO: Starting evaluation with evaluator: sheets_cell_values\\n'\n", + " \"INFO: Evaluator args: [{'A1': 'ABC', 'B1': '-0.08'}]\\n\"\n", + " 'INFO: Partial rewarding: False\\n'\n", + " 'INFO: Starting sheets_cell_values evaluation for environment: '\n", + " 'af7a34a0-43b0-44d2-82d0-2b66ed16f1ea\\n'\n", + " \"INFO: Raw args received: [{'A1': 'ABC', 'B1': '-0.08'}] (type: \"\n", + " \")\\n\"\n", + " 'INFO: Partial rewarding enabled: False\\n'\n", + " 'INFO: === Google Sheets Cell Value Verification ===\\n'\n", + " 'INFO: Current page URL: '\n", + " 'https://docs.google.com/spreadsheets/d/1h-Ec3rW9sAME2sTn8qxIvFxO6qXtdURPacEFL5DJnqw/edit?gid=700326861#gid=700326861\\n'\n", + " 'INFO: ✅ Confirmed on Google Sheets page\\n'\n", + " 'INFO: Processing args parameter...\\n'\n", + " 'INFO: Args is a list with 1 items, extracting first item\\n'\n", + " \"INFO: Extracted: {'A1': 'ABC', 'B1': '-0.08'} (type: )\\n\"\n", + " 'INFO: Cell checks to perform: 2 cells\\n'\n", + " \"INFO: A1 -> expected: 'ABC'\\n\"\n", + " \"INFO: B1 -> expected: '-0.08'\\n\"\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " \"sheets_cell_values: Checking cells: {'A1': 'ABC', 'B1': '-0.08'}\\n\"\n", + " 'INFO: === ANSWER Sheet Navigation ===\\n'\n", + " 'INFO: Attempt 1/3: Attempting to find and navigate to ANSWER sheet '\n", + " 'tab...\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Attempt 1/3: Attempting to navigate to ANSWER '\n", + " 'sheet\\n'\n", + " 'INFO: Searching for ANSWER tab with selector: '\n", + " 'span.docs-sheet-tab-name:has-text(\"ANSWER\")\\n'\n", + " 'INFO: ANSWER tab search result (attempt 1): Found\\n'\n", + " 'INFO: ✅ Found ANSWER sheet tab on attempt 1, clicking on it...\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Found ANSWER sheet tab on attempt 1, clicking on '\n", + " 'it\\n'\n", + " 'ERROR: ❌ Error navigating to ANSWER sheet on attempt 1: '\n", + " 'Locator.click: Timeout 30000ms exceeded.\\n'\n", + " 'Call log:\\n'\n", + " ' - waiting for '\n", + " 'locator(\"span.docs-sheet-tab-name:has-text(\\\\\"ANSWER\\\\\")\")\\n'\n", + " ' - - locator resolved to ANSWER\\n'\n", + " ' - - attempting click action\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 20ms\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 100ms\\n'\n", + " ' - 35 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 500ms\\n'\n", + " '\\n'\n", + " 'WARNING: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Error navigating to ANSWER sheet on attempt 1: '\n", + " 'Locator.click: Timeout 30000ms exceeded.\\n'\n", + " 'Call log:\\n'\n", + " ' - waiting for '\n", + " 'locator(\"span.docs-sheet-tab-name:has-text(\\\\\"ANSWER\\\\\")\")\\n'\n", + " ' - - locator resolved to ANSWER\\n'\n", + " ' - - attempting click action\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 20ms\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 100ms\\n'\n", + " ' - 35 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 500ms\\n'\n", + " '\\n'\n", + " 'INFO: Waiting 500ms before retry 2...\\n'\n", + " 'INFO: Attempt 2/3: Attempting to find and navigate to ANSWER sheet '\n", + " 'tab...\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Attempt 2/3: Attempting to navigate to ANSWER '\n", + " 'sheet\\n'\n", + " 'INFO: Searching for ANSWER tab with selector: '\n", + " 'span.docs-sheet-tab-name:has-text(\"ANSWER\")\\n'\n", + " 'INFO: ANSWER tab search result (attempt 2): Found\\n'\n", + " 'INFO: ✅ Found ANSWER sheet tab on attempt 2, clicking on it...\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Found ANSWER sheet tab on attempt 2, clicking on '\n", + " 'it\\n'\n", + " 'ERROR: ❌ Error navigating to ANSWER sheet on attempt 2: '\n", + " 'Locator.click: Timeout 30000ms exceeded.\\n'\n", + " 'Call log:\\n'\n", + " ' - waiting for '\n", + " 'locator(\"span.docs-sheet-tab-name:has-text(\\\\\"ANSWER\\\\\")\")\\n'\n", + " ' - - locator resolved to ANSWER\\n'\n", + " ' - - attempting click action\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 20ms\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 100ms\\n'\n", + " ' - 35 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 500ms\\n'\n", + " '\\n'\n", + " 'WARNING: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Error navigating to ANSWER sheet on attempt 2: '\n", + " 'Locator.click: Timeout 30000ms exceeded.\\n'\n", + " 'Call log:\\n'\n", + " ' - waiting for '\n", + " 'locator(\"span.docs-sheet-tab-name:has-text(\\\\\"ANSWER\\\\\")\")\\n'\n", + " ' - - locator resolved to ANSWER\\n'\n", + " ' - - attempting click action\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 20ms\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 100ms\\n'\n", + " ' - 35 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 500ms\\n'\n", + " '\\n'\n", + " 'INFO: Waiting 500ms before retry 3...\\n'\n", + " 'INFO: Attempt 3/3: Attempting to find and navigate to ANSWER sheet '\n", + " 'tab...\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Attempt 3/3: Attempting to navigate to ANSWER '\n", + " 'sheet\\n'\n", + " 'INFO: Searching for ANSWER tab with selector: '\n", + " 'span.docs-sheet-tab-name:has-text(\"ANSWER\")\\n'\n", + " 'INFO: ANSWER tab search result (attempt 3): Found\\n'\n", + " 'INFO: ✅ Found ANSWER sheet tab on attempt 3, clicking on it...\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Found ANSWER sheet tab on attempt 3, clicking on '\n", + " 'it\\n'\n", + " 'ERROR: ❌ Error navigating to ANSWER sheet on attempt 3: '\n", + " 'Locator.click: Timeout 30000ms exceeded.\\n'\n", + " 'Call log:\\n'\n", + " ' - waiting for '\n", + " 'locator(\"span.docs-sheet-tab-name:has-text(\\\\\"ANSWER\\\\\")\")\\n'\n", + " ' - - locator resolved to ANSWER\\n'\n", + " ' - - attempting click action\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 20ms\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 100ms\\n'\n", + " ' - 35 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 500ms\\n'\n", + " '\\n'\n", + " 'WARNING: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Error navigating to ANSWER sheet on attempt 3: '\n", + " 'Locator.click: Timeout 30000ms exceeded.\\n'\n", + " 'Call log:\\n'\n", + " ' - waiting for '\n", + " 'locator(\"span.docs-sheet-tab-name:has-text(\\\\\"ANSWER\\\\\")\")\\n'\n", + " ' - - locator resolved to ANSWER\\n'\n", + " ' - - attempting click action\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 20ms\\n'\n", + " ' - 2 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 100ms\\n'\n", + " ' - 35 × waiting for element to be visible, enabled and stable\\n'\n", + " ' - - element is visible, enabled and stable\\n'\n", + " ' - - scrolling into view if needed\\n'\n", + " ' - - done scrolling\\n'\n", + " ' - -
'\n", + " 'intercepts pointer events\\n'\n", + " ' - - retrying click action\\n'\n", + " ' - - waiting 500ms\\n'\n", + " '\\n'\n", + " 'WARNING: ⚠️ Failed to navigate to ANSWER sheet after 3 attempts, '\n", + " 'proceeding with current sheet\\n'\n", + " 'WARNING: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Failed to navigate to ANSWER sheet after 3 '\n", + " 'attempts, proceeding with current sheet\\n'\n", + " 'INFO: === File Content Extraction ===\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Granted read-write permissions\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Extracting page contents\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Selecting content\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Successfully extracted 157940 characters from '\n", + " 'file\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Found 5003 rows in content\\n'\n", + " 'INFO: Content extracted: 157940 characters\\n'\n", + " 'INFO: === Cell Content Parsing ===\\n'\n", + " 'INFO: Split file content into 5003 rows\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Found 5003 rows in content\\n'\n", + " 'INFO: First few rows of content:\\n'\n", + " \"INFO: Row 1: 'TradeDate | Ticker | ClosePrice | Volume | | '\\n\"\n", + " \"INFO: Row 2: '2023-01-02 | ABC | 476.87 | 2225355 | | '\\n\"\n", + " \"INFO: Row 3: '2023-01-02 | DEF | 322.21 | 3778582 | | '\\n\"\n", + " 'INFO: ... and 5000 more rows\\n'\n", + " 'INFO: === Cell Reference Parsing ===\\n'\n", + " \"INFO: Processing cell reference: 'A1' -> expected: 'ABC'\\n\"\n", + " \"INFO: Parsed 'A1' -> row=1 (0-indexed: 0), col=A (0-indexed: 0)\\n\"\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Parsed cell A1 as row=0, col=0\\n'\n", + " 'INFO: Row 1 exists in content\\n'\n", + " \"INFO: Row 1 has 6 columns: ['Col1', 'Col2', 'Col3', 'Col4', \"\n", + " \"'Col5', 'Col6']\\n\"\n", + " \"INFO: ✅ Found value for A1: 'TradeDate'\\n\"\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " \"sheets_cell_values: Found value for A1: 'TradeDate'\\n\"\n", + " \"INFO: Processing cell reference: 'B1' -> expected: '-0.08'\\n\"\n", + " \"INFO: Parsed 'B1' -> row=1 (0-indexed: 0), col=B (0-indexed: 1)\\n\"\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Parsed cell B1 as row=0, col=1\\n'\n", + " 'INFO: Row 1 exists in content\\n'\n", + " \"INFO: Row 1 has 6 columns: ['Col1', 'Col2', 'Col3', 'Col4', \"\n", + " \"'Col5', 'Col6']\\n\"\n", + " \"INFO: ✅ Found value for B1: 'Ticker'\\n\"\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " \"sheets_cell_values: Found value for B1: 'Ticker'\\n\"\n", + " 'INFO: === Cell Value Comparison ===\\n'\n", + " 'INFO: Comparing cell A1:\\n'\n", + " \"INFO: Expected: 'ABC' (type: )\\n\"\n", + " \"INFO: Actual: 'TradeDate' (type: )\\n\"\n", + " \"INFO: ❌ VALUE MISMATCH: 'TradeDate' != 'ABC'\\n\"\n", + " 'INFO: Comparing cell B1:\\n'\n", + " \"INFO: Expected: '-0.08' (type: )\\n\"\n", + " \"INFO: Actual: 'Ticker' (type: )\\n\"\n", + " \"INFO: ❌ VALUE MISMATCH: 'Ticker' != '-0.08'\\n\"\n", + " 'INFO: === Final Results ===\\n'\n", + " 'INFO: Cell comparison summary:\\n'\n", + " 'INFO: Total cells checked: 2\\n'\n", + " 'INFO: Matches: 0\\n'\n", + " 'INFO: Mismatches: 2\\n'\n", + " \"INFO: Failed cells: ['A1:', 'B1:']\\n\"\n", + " 'INFO: ❌ NOT all cells match expected values\\n'\n", + " 'INFO: Mismatches: [\"Cell A1: expected \\'ABC\\', got \\'TradeDate\\'\", '\n", + " '\"Cell B1: expected \\'-0.08\\', got \\'Ticker\\'\"]\\n'\n", + " 'INFO: [TASK af7a34a0-43b0-44d2-82d0-2b66ed16f1ea] '\n", + " 'sheets_cell_values: Mismatches found: [\"Cell A1: expected \\'ABC\\', '\n", + " 'got \\'TradeDate\\'\", \"Cell B1: expected \\'-0.08\\', got \\'Ticker\\'\"]\\n'\n", + " 'INFO: Final reward: 0.0\\n'\n", + " 'INFO: === Sheets Cell Values Evaluation Complete ===\\n'\n", + " 'INFO: Evaluation completed. Final reward: 0.0\\n',\n", + " 'reward': 0.0}\n" + ] + } + ], "source": [ "# Evaluate environment state\n", "result = await env.evaluate()\n", @@ -292,7 +742,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -318,17 +768,426 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%|----------------------------------------| 0/200 [1:24