ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter9/gaia-experience/AWorld/examples/browser_use/README.md

@@ -0,0 +1,4 @@
+# Browser use
+
+Agents specialized in web browser automation.
+The implementation of browser agent version is now derived from [browser use](https://github.com/browser-use/browser-use), which we have made a lot of modifications to integrate into our own framework

chapter9/gaia-experience/AWorld/examples/browser_use/__init__.py

@@ -0,0 +1,2 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.

chapter9/gaia-experience/AWorld/examples/browser_use/agent.py

@@ -0,0 +1,565 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+import re
+import time
+import traceback
+import json
+from pathlib import Path
+from typing import Dict, Any, Optional, List, Union, Tuple
+from dataclasses import dataclass, field
+
+from langchain_core.messages import HumanMessage, BaseMessage, AIMessage, ToolMessage
+from pydantic import ValidationError
+
+from aworld.core.agent.base import AgentFactory, AgentResult
+from aworld.agents.llm_agent import Agent
+from examples.browser_use.prompts import SystemPrompt
+from examples.browser_use.utils import convert_input_messages, extract_json_from_model_output, estimate_messages_tokens
+from examples.browser_use.common import AgentState, AgentStepInfo, AgentHistory, PolicyMetadata, AgentBrain
+from aworld.config.conf import AgentConfig, ConfigDict
+from aworld.core.common import Observation, ActionModel, ToolActionInfo, ActionResult
+from aworld.logs.util import logger
+from examples.browser_use.prompts import AgentMessagePrompt
+from examples.common.tools.common import Tools
+from examples.common.tools.tool_action import BrowserAction
+
+
+@dataclass
+class Trajectory:
+    """A class to store agent history records, including all observations, info and AgentResult"""
+    history: List[tuple[List[BaseMessage], Observation, Dict[str, Any], AIMessage, AgentResult]] = field(
+        default_factory=list)
+
+    def add_step(self, input_messages: List[BaseMessage], observation: Observation, info: Dict[str, Any],
+                 output_message: AIMessage, agent_result: AgentResult):
+        """Add a step to the history"""
+        self.history.append((input_messages, observation, info, output_message, agent_result))
+
+    def get_history(self) -> List[tuple[List[BaseMessage], Observation, Dict[str, Any], AIMessage, AgentResult]]:
+        """Get the complete history"""
+        return self.history
+
+    def save_history(self, file_path: str):
+        his_li = []
+        for input_messages, observation, info, output_message, agent_result in self.get_history():
+            llm_input = [{"type": input_message.type, "content": input_message.content} for input_message in
+                         input_messages]
+            llm_output = output_message.content
+            his_li.append({"llm_input": llm_input, "llm_output": llm_output})
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(his_li, f, ensure_ascii=False, indent=4)
+
+
+class BrowserAgent(Agent):
+    def __init__(self, conf: Union[Dict[str, Any], ConfigDict, AgentConfig], name: str, **kwargs):
+        super(BrowserAgent, self).__init__(conf=conf, name=name, **kwargs)
+        self.state = AgentState()
+        self.settings = self.conf
+        provider = self.conf.llm_config.llm_provider
+        if self.conf.llm_config.llm_provider:
+            self.conf.llm_config.llm_provider = "chat" + provider
+        else:
+            raise Exception("no llm provider")
+
+        self.save_file_path = self.conf.save_file_path
+        self.available_actions = self._build_action_prompt()
+        # Note: Removed _message_manager initialization as it's no longer used
+        # Initialize trajectory
+        self.trajectory = Trajectory()
+        self._init = False
+
+    def reset(self, options: Dict[str, Any] = None):
+        super(BrowserAgent, self).reset(options)
+
+        # Reset trajectory
+        self.trajectory = Trajectory()
+
+        # Note: Removed _message_manager initialization as it's no longer used
+        # _estimate_tokens_for_messages method now directly uses functions from utils.py
+
+        self._init = True
+
+    def _build_action_prompt(self) -> str:
+        def _prompt(info: ToolActionInfo) -> str:
+            s = f'{info.desc}: \n'
+            s += '{' + str(info.name) + ': '
+            if info.input_params:
+                s += str({k: {"title": k, "type": v.type} for k, v in info.input_params.items()})
+            s += '}'
+            return s
+
+        val = "\n".join([_prompt(v.value) for k, v in BrowserAction.__members__.items()])
+        return val
+
+    def _log_message_sequence(self, input_messages: List[BaseMessage]) -> None:
+        """Log the sequence of messages for debugging purposes"""
+        logger.info(f"[agent] 🔍 Invoking LLM with {len(input_messages)} messages")
+        logger.info("[agent] 📝 Messages sequence:")
+        for i, msg in enumerate(input_messages):
+            prefix = msg.type
+            logger.info(f"[agent] Message {i + 1}: {prefix} ===================================")
+            if isinstance(msg.content, list):
+                for item in msg.content:
+                    if item.get('type') == 'text':
+                        logger.info(f"[agent] Text content: {item.get('text')}")
+                    elif item.get('type') == 'image_url':
+                        # Only print the first 30 characters of image URL to avoid printing entire base64
+                        image_url = item.get('image_url', {}).get('url', '')
+                        if image_url.startswith('data:image'):
+                            logger.info(f"[agent] Image: [Base64 image data]")
+                        else:
+                            logger.info(f"[agent] Image URL: {image_url[:30]}...")
+            else:
+                content = str(msg.content)
+                chunk_size = 500
+                for j in range(0, len(content), chunk_size):
+                    chunk = content[j:j + chunk_size]
+                    if j == 0:
+                        logger.info(f"[agent] Content: {chunk}")
+                    else:
+                        logger.info(f"[agent] Content (continued): {chunk}")
+
+            if isinstance(msg, AIMessage) and hasattr(msg, 'tool_calls') and msg.tool_calls:
+                for tool_call in msg.tool_calls:
+                    logger.info(f"[agent] Tool call: {tool_call.get('name')} - ID: {tool_call.get('id')}")
+                    args = str(tool_call.get('args', {}))[:1000]
+                    logger.info(f"[agent] Tool args: {args}...")
+
+    def save_process(self, file_path: str):
+        self.trajectory.save_history(file_path)
+
+    def policy(self,
+               observation: Observation,
+               info: Dict[str, Any] = None, **kwargs) -> Union[List[ActionModel], None]:
+        start_time = time.time()
+
+        if self._init is False:
+            self.reset({"task": observation.content})
+
+        self._finished = False
+        # Save current observation to state for message construction
+        self.state.last_result = observation.action_result
+
+        if self.conf.max_steps <= self.state.n_steps:
+            logger.info('Last step finishing up')
+
+        logger.info(f'[agent] step {self.state.n_steps}')
+
+        # Use the new method to build messages, passing the current observation
+        input_messages = self.build_messages_from_trajectory_and_observation(observation=observation)
+
+        # Note: Special message addition has been moved to build_messages_from_trajectory_and_observation
+
+        # Estimate token count
+        tokens = self._estimate_tokens_for_messages(input_messages)
+
+        llm_result = None
+        output_message = None
+        try:
+            # Log the message sequence
+            self._log_message_sequence(input_messages)
+
+            output_message, llm_result = self._do_policy(input_messages)
+
+            if not llm_result:
+                logger.error("[agent] ❌ Failed to parse LLM response")
+                return [ActionModel(tool_name=Tools.BROWSER.value, action_name="stop")]
+
+            self.state.n_steps += 1
+
+            # No longer need to remove the last state message
+            # self._message_manager._remove_last_state_message()
+
+            if self.state.stopped or self.state.paused:
+                logger.info('Browser gent paused after getting state')
+                return [ActionModel(tool_name=Tools.BROWSER.value, action_name="stop")]
+
+            tool_action = llm_result.actions
+
+            # Add the current step to the trajectory
+            self.trajectory.add_step(input_messages, observation, info, output_message, llm_result)
+
+        except Exception as e:
+            logger.warning(traceback.format_exc())
+            # No longer need to remove the last state message
+            # self._message_manager._remove_last_state_message()
+            logger.error(f"[agent] ❌ Error parsing LLM response: {str(e)}")
+
+            # Create an AgentResult object with an empty actions list
+            error_result = AgentResult(
+                current_state=AgentBrain(
+                    evaluation_previous_goal="Failed due to error",
+                    memory=f"Error occurred: {str(e)}",
+                    thought="Recover from error",
+                    next_goal="Recover from error"
+                ),
+                actions=[]  # Empty actions list
+            )
+
+            # Add the error state to the trajectory
+            self.trajectory.add_step(input_messages, observation, info, output_message, error_result)
+
+            raise RuntimeError("Browser agent encountered exception while making the policy.", e)
+        finally:
+            if llm_result:
+                # Only keep the history_item creation part
+                metadata = PolicyMetadata(
+                    number=self.state.n_steps,
+                    start_time=start_time,
+                    end_time=time.time(),
+                    input_tokens=tokens,
+                )
+                self._make_history_item(llm_result, observation, observation.action_result, metadata)
+            else:
+                logger.warning("no result to record!")
+
+        return tool_action
+
+    def _do_policy(self, input_messages: list[BaseMessage]) -> Tuple[AIMessage, AgentResult]:
+        THINK_TAGS = re.compile(r'<think>.*?</think>', re.DOTALL)
+
+        def _remove_think_tags(text: str) -> str:
+            """Remove think tags from text"""
+            return re.sub(THINK_TAGS, '', text)
+
+        input_messages = self._convert_input_messages(input_messages)
+        output_message = None
+        try:
+
+            output_message = self.llm.invoke(input_messages)
+
+            if not output_message or not output_message.content:
+                logger.warning("[agent] LLM returned empty response")
+                return output_message, AgentResult(
+                    current_state=AgentBrain(evaluation_previous_goal="", memory="", thought="", next_goal=""),
+                    actions=[ActionModel(agent_name=self.id(), tool_name='browser', action_name="stop")])
+        except:
+            logger.error(f"[agent] Response content: {output_message}")
+            raise RuntimeError('call llm fail, please check llm conf and network.')
+
+        if self.model_name == 'deepseek-reasoner':
+            output_message.content = _remove_think_tags(output_message.content)
+        try:
+            # Get max retries from config
+            max_retries = self.settings.get('max_llm_json_retries', 3)
+            retry_count = 0
+            json_parse_error = None
+
+            while retry_count < max_retries:
+                try:
+                    parsed_json = extract_json_from_model_output(output_message.content)
+                    # If parsing succeeds, break out of the retry loop
+                    json_parse_error = None
+                    break
+                except ValueError as e:
+                    # Store the error and retry
+                    json_parse_error = e
+                    retry_count += 1
+                    logger.warning(f"[agent] Failed to parse JSON (attempt {retry_count}/{max_retries}): {str(e)}")
+
+                    if retry_count < max_retries:
+                        # Add a reminder message about JSON format with specific structure guidance
+                        format_reminder = HumanMessage(
+                            content="Your responses must be always JSON with the specified format. Make sure your response includes a 'current_state' object with 'evaluation_previous_goal', 'memory', and 'next_goal' fields, and an 'action' array with the actions to perform. Do not include any explanatory text, only return the raw JSON.")
+                        retry_messages = input_messages.copy()
+                        retry_messages.append(format_reminder)
+
+                        # Retry with the updated messages
+                        logger.info(
+                            f"[agent] Retrying LLM invocation ({retry_count}/{max_retries}) with format reminder")
+                        output_message = self.llm.invoke(retry_messages)
+
+                        # Check for empty response during retry
+                        if not output_message or not output_message.content:
+                            logger.warning(
+                                f"[agent] LLM returned empty response on retry attempt {retry_count}/{max_retries}")
+                            # Continue to next retry instead of immediately returning
+                            continue
+
+                        if self.model_name == 'deepseek-reasoner':
+                            output_message.content = _remove_think_tags(output_message.content)
+
+            # If all retries failed, raise the last error
+            if json_parse_error:
+                logger.error(f"[agent] ❌ All {max_retries} attempts to parse JSON failed")
+                raise json_parse_error
+
+            logger.info((f"llm response: {parsed_json}"))
+            try:
+                agent_brain = AgentBrain(**parsed_json['current_state'])
+            except:
+                agent_brain = None
+            actions = parsed_json.get('action')
+            result = []
+            if not actions:
+                actions = parsed_json.get("actions")
+            if not actions:
+                logger.warning("agent not policy  an action.")
+                self._finished = True
+                return output_message, AgentResult(current_state=agent_brain,
+                                                   actions=[ActionModel(tool_name='browser',
+                                                                        agent_name=self.id(),
+                                                                        action_name="done")])
+
+            for action in actions:
+                if "action_name" in action:
+                    action_name = action['action_name']
+                    browser_action = BrowserAction.get_value_by_name(action_name)
+                    if not browser_action:
+                        logger.warning(f"Unsupported action: {action_name}")
+                    if action_name == "done":
+                        self._finished = True
+                    action_model = ActionModel(agent_name=self.id(),
+                                               tool_name='browser',
+                                               action_name=action_name,
+                                               params=action.get('params', {}))
+                    result.append(action_model)
+                else:
+                    for k, v in action.items():
+                        browser_action = BrowserAction.get_value_by_name(k)
+                        if not browser_action:
+                            logger.warning(f"Unsupported action: {k}")
+
+                        action_model = ActionModel(agent_name=self.id(), tool_name='browser', action_name=k, params=v)
+                        result.append(action_model)
+                        if k == "done":
+                            self._finished = True
+            return output_message, AgentResult(current_state=agent_brain, actions=result)
+        except (ValueError, ValidationError) as e:
+            logger.warning(f'Failed to parse model output: {output_message} {str(e)}')
+            raise ValueError('Could not parse response.')
+
+    def _convert_input_messages(self, input_messages: list[BaseMessage]) -> list[BaseMessage]:
+        """Convert input messages to the correct format"""
+        if self.model_name == 'deepseek-reasoner' or self.model_name.startswith('deepseek-r1'):
+            return convert_input_messages(input_messages, self.model_name)
+        else:
+            return input_messages
+
+    def _make_history_item(self,
+                           model_output: AgentResult | None,
+                           state: Observation,
+                           result: list[ActionResult],
+                           metadata: Optional[PolicyMetadata] = None) -> None:
+        content = ""
+        if hasattr(state, 'dom_tree') and state.dom_tree is not None:
+            if hasattr(state.dom_tree, 'element_tree'):
+                content = state.dom_tree.element_tree.__repr__()
+            else:
+                content = str(state.dom_tree)
+
+        history_item = AgentHistory(model_output=model_output,
+                                    result=state.action_result,
+                                    metadata=metadata,
+                                    content=content,
+                                    base64_img=state.image if hasattr(state, 'image') else None)
+
+        self.state.history.history.append(history_item)
+
+    def _process_action_result(self, action_result, messages, tool_call=None):
+        """Helper method to process an action result and add appropriate messages"""
+        if action_result.content is not None:
+            messages.append(HumanMessage(content='Action result: ' + action_result.content))
+        elif action_result.error is not None:
+            # Assemble error message when error information exists
+            messages.append(HumanMessage(content='Action result: ' + action_result.error))
+            if tool_call is not None:
+                logger.warning(f"Action {tool_call} failed: {action_result.error}")
+            else:
+                logger.warning(f"Action failed: {action_result.error}")
+            # If there is an error but success is true, log the error and terminate the program as the result is invalid
+            if action_result.success is True:
+                error_msg = f"Invalid result: success=True but error message exists: {action_result.error}"
+                logger.error(error_msg)
+                raise ValueError(error_msg)
+        return action_result.error is not None
+
+    def build_messages_from_trajectory_and_observation(self, observation: Optional[Observation] = None) -> List[
+        BaseMessage]:
+        """
+        Build complete message history from trajectory and current observation
+        
+        Args:
+            observation: Current observation object, if None current observation won't be added
+        """
+        messages = []
+        # Add system message
+        system_message = SystemPrompt(
+            max_actions_per_step=self.settings.get('max_actions_per_step')
+        ).get_system_message()
+        if isinstance(system_message, tuple):
+            system_message = system_message[0]
+        messages.append(system_message)
+
+        tool_calling_method = self.settings.get("tool_calling_method")
+        llm_provider = self.conf.llm_config.llm_provider
+
+        if tool_calling_method == 'raw' or (tool_calling_method == 'auto' and (
+                llm_provider == 'deepseek-reasoner' or llm_provider.startswith('deepseek-r1'))):
+            message_context = f'\n\nAvailable actions: {self.available_actions}'
+        else:
+            message_context = None
+
+        # Add task context (if any)
+        if message_context:
+            context_message = HumanMessage(content='Context for the task' + message_context)
+            messages.append(context_message)
+
+        # Add task message
+        task_message = HumanMessage(
+            content=f'Your ultimate task is: """{self.task}""". If you achieved your ultimate task, stop everything and use the done action in the next step to complete the task. If not, continue as usual.'
+        )
+        messages.append(task_message)
+
+        # Add example output
+        placeholder_message = HumanMessage(content='Example output:')
+        messages.append(placeholder_message)
+
+        # Add example tool call
+        tool_calls = [
+            {
+                'name': 'AgentOutput',
+                'args': {
+                    'current_state': {
+                        'evaluation_previous_goal': 'Success - I opend the first page',
+                        'memory': 'Starting with the new task. I have completed 1/10 steps',
+                        'thought': 'From the current page I can get information about all the companies.',
+                        'next_goal': 'Click on company a',
+                    },
+                    'action': [{'click_element': {'index': 0}}],
+                },
+                'id': '1',
+                'type': 'tool_call',
+            }
+        ]
+        example_tool_call = AIMessage(
+            content='',
+            tool_calls=tool_calls,
+        )
+        messages.append(example_tool_call)
+
+        # Add first tool message with "Browser started" content
+        messages.append(ToolMessage(content='Browser started', tool_call_id='1'))
+
+        # Add task history marker
+        messages.append(HumanMessage(content='[Your task history memory starts here]'))
+
+        # Add available file paths (if any)
+        if self.settings.get('available_file_paths'):
+            filepaths_msg = HumanMessage(
+                content=f'Here are file paths you can use: {self.settings.get("available_file_paths")}')
+            messages.append(filepaths_msg)
+        previous_action_entries = []
+        # Add messages from the history trajectory
+        for input_msgs, obs, info, output_msg, llm_result in self.trajectory.get_history():
+            # Check the previous step's actionResult
+            has_error = False
+            if obs.action_result is not None:
+                # The previous action entries should match with action results
+                if len(previous_action_entries) == 0:
+                    # if previous_action_entries is empty，process action_result directly
+                    logger.info(
+                        f"History item with action_result count ({len(obs.action_result)}) with empty previous actions - skipping count check")
+                elif len(previous_action_entries) == len(obs.action_result):
+                    for i, one_action_result in enumerate(obs.action_result):
+                        has_error = self._process_action_result(one_action_result, messages,
+                                                                previous_action_entries[i]) or has_error
+                else:
+                    # If sizes don't match, this is a critical error
+                    error_msg = f"Action results count ({len(obs.action_result)}) doesn't match action entries count ({len(previous_action_entries)})"
+                    logger.error(error_msg)
+                    has_error = True
+                    # raise ValueError(error_msg)
+
+            # Add agent response
+            if llm_result:
+                # Create AI message
+                output_data = llm_result.model_dump(mode='json', exclude_unset=True)
+                action_entries = [{action.action_name: action.params} for action in llm_result.actions]
+                output_data["action"] = action_entries
+                if "actions" in output_data:
+                    del output_data["actions"]
+
+                # Calculate tool_id based on trajectory history. If no actions yet, start with ID 1
+                tool_id = 1 if len(self.trajectory.get_history()) == 0 else len(self.trajectory.get_history()) + 1
+                tool_calls = [
+                    {
+                        'name': 'AgentOutput',
+                        'args': output_data,
+                        'id': str(tool_id),
+                        'type': 'tool_call',
+                    }
+                ]
+                previous_action_entries = action_entries
+                ai_message = AIMessage(
+                    content='',
+                    tool_calls=tool_calls,
+                )
+                messages.append(ai_message)
+
+                # Add empty tool message after each AIMessage
+                messages.append(ToolMessage(content='', tool_call_id=str(tool_id)))
+
+        # Add current observation - using the passed observation parameter instead of self.state.current_observation
+        if observation:
+            # Check if the current observation has an action_result with error
+            has_error = False
+            if hasattr(observation, 'action_result') and observation.action_result is not None:
+                # Match action results with previous actions
+                if len(previous_action_entries) == 0:
+                    # if previous_action_entries is empty，process action_result directly
+                    logger.info(
+                        f"Current observation with action_result count ({len(observation.action_result)}) with empty previous actions - skipping count check")
+                elif len(previous_action_entries) == len(observation.action_result):
+                    for i, one_action_result in enumerate(observation.action_result):
+                        has_error = self._process_action_result(one_action_result, messages,
+                                                                previous_action_entries[i]) or has_error
+                else:
+                    # If sizes don't match, this is a critical error
+                    error_msg = f"Action results count ({len(observation.action_result)}) doesn't match action entries count ({len(previous_action_entries)})"
+                    logger.error(error_msg)
+                    has_error = True
+
+            # If there's an error, append observation content outside the loop
+            if has_error and observation.content:
+                messages.append(HumanMessage(content=observation.content))
+            # If no error, process the observation normally
+            elif not has_error:
+                step_info = AgentStepInfo(number=self.state.n_steps, max_steps=self.conf.max_steps)
+                if hasattr(observation, 'dom_tree') and observation.dom_tree:
+                    state_message = AgentMessagePrompt(
+                        observation,
+                        self.state.last_result,
+                        include_attributes=self.settings.get('include_attributes',
+                                                             ["title", "type", "name", "role", "aria-label",
+                                                              "placeholder", "value", "alt", "aria-expanded",
+                                                              "data-date-format"]),
+                        step_info=step_info,
+                    ).get_user_message(self.settings.get('use_vision'))
+                    messages.append(state_message)
+                elif observation.content:
+                    messages.append(HumanMessage(content=observation.content))
+
+        # Add special message for the last step
+        # Note: Moved here from policy method to centralize all message building logic
+        if self.conf.max_steps <= self.state.n_steps:
+            last_step_message = f"""
+                Now comes your last step. Use only the "done" action now. No other actions - so here your action sequence must have length 1.
+                \nIf the task is not yet fully finished as requested by the user, set success in "done" to false! E.g. if not all steps are fully completed.
+                \nIf the task is fully finished, set success in "done" to true.
+                \nInclude everything you found out for the ultimate task in the done text.
+            """
+            messages.append(HumanMessage(content=[{'type': 'text', 'text': last_step_message}]))
+
+        return messages
+
+    def _estimate_tokens_for_messages(self, messages: List[BaseMessage]) -> int:
+        """Roughly estimate token count for message list"""
+        # Note: Using estimate_messages_tokens function from utils.py instead of calling _message_manager
+        # This decouples the dependency on MessageManager
+        return estimate_messages_tokens(
+            messages,
+            image_tokens=self.settings.get('image_tokens', 800),
+            estimated_characters_per_token=self.settings.get('estimated_characters_per_token', 3)
+        )

chapter9/gaia-experience/AWorld/examples/browser_use/common.py

@@ -0,0 +1,130 @@
+# coding: utf-8
+
+import json
+import traceback
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional, Dict, List
+
+from openai import RateLimitError
+from pydantic import BaseModel, ConfigDict, Field
+
+from aworld.core.common import ActionResult
+
+
+class PolicyMetadata(BaseModel):
+    """Metadata for a single step including timing information"""
+    start_time: float
+    end_time: float
+    number: int
+    input_tokens: int
+
+    @property
+    def duration_seconds(self) -> float:
+        """Calculate step duration in seconds"""
+        return self.end_time - self.start_time
+
+
+class AgentBrain(BaseModel):
+    """Current state of the agent"""
+    evaluation_previous_goal: str = None
+    memory: str = None
+    thought: str = None
+    next_goal: str = None
+
+
+class AgentHistory(BaseModel):
+    """History item for agent actions"""
+    model_output: Optional[BaseModel] = None
+    result: List[ActionResult]
+    metadata: Optional[PolicyMetadata] = None
+    content: Optional[str] = None
+    base64_img: Optional[str] = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    def model_dump(self, **kwargs) -> Dict[str, Any]:
+        """Custom serialization handling"""
+        return {
+            'model_output': self.model_output.model_dump() if self.model_output else None,
+            'result': [r.model_dump(exclude_none=True) for r in self.result],
+            'metadata': self.metadata.model_dump() if self.metadata else None,
+            'content': self.xml_content,
+            'base64_img': self.base64_img
+        }
+
+
+class AgentHistoryList(BaseModel):
+    """List of agent history items"""
+    history: List[AgentHistory]
+
+    def total_duration_seconds(self) -> float:
+        """Get total duration of all steps in seconds"""
+        total = 0.0
+        for h in self.history:
+            if h.metadata:
+                total += h.metadata.duration_seconds
+        return total
+
+    def save_to_file(self, filepath: str | Path) -> None:
+        """Save history to JSON file with proper serialization"""
+        try:
+            Path(filepath).parent.mkdir(parents=True, exist_ok=True)
+            data = self.model_dump()
+            with open(filepath, 'w', encoding='utf-8') as f:
+                json.dump(data, f, indent=2)
+        except Exception as e:
+            raise e
+
+    def model_dump(self, **kwargs) -> Dict[str, Any]:
+        """Custom serialization that properly uses AgentHistory's model_dump"""
+        return {
+            'history': [h.model_dump(**kwargs) for h in self.history],
+        }
+
+    @classmethod
+    def load_from_file(cls, filepath: str | Path) -> 'AgentHistoryList':
+        """Load history from JSON file"""
+        with open(filepath, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return cls.model_validate(data)
+
+
+class AgentError:
+    """Container for agent error handling"""
+    VALIDATION_ERROR = 'Invalid model output format. Please follow the correct schema.'
+    RATE_LIMIT_ERROR = 'Rate limit reached. Waiting before retry.'
+    NO_VALID_ACTION = 'No valid action found'
+
+    @staticmethod
+    def format_error(error: Exception, include_trace: bool = False) -> str:
+        """Format error message based on error type and optionally include trace"""
+        if isinstance(error, RateLimitError):
+            return AgentError.RATE_LIMIT_ERROR
+        if include_trace:
+            return f'{str(error)}\nStacktrace:\n{traceback.format_exc()}'
+        return f'{str(error)}'
+
+
+class AgentState(BaseModel):
+    """Holds all state information for an Agent"""
+
+    agent_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    n_steps: int = 1
+    consecutive_failures: int = 0
+    last_result: Optional[List['ActionResult']] = None
+    history: AgentHistoryList = Field(default_factory=lambda: AgentHistoryList(history=[]))
+    last_plan: Optional[str] = None
+    paused: bool = False
+    stopped: bool = False
+
+
+@dataclass
+class AgentStepInfo:
+    number: int
+    max_steps: int
+
+    def is_last_step(self) -> bool:
+        """Check if this is the last step"""
+        return self.number >= self.max_steps - 1

chapter9/gaia-experience/AWorld/examples/browser_use/config.py

@@ -0,0 +1,25 @@
+# coding: utf-8
+
+from typing import Optional
+from aworld.config.conf import AgentConfig
+from typing import Literal
+
+ToolCallingMethod = Literal['function_calling', 'json_mode', 'raw', 'auto']
+
+
+class BrowserAgentConfig(AgentConfig):
+    use_vision: bool = True
+    use_vision_for_planner: bool = False
+    save_conversation_path: Optional[str] = None
+    save_conversation_path_encoding: Optional[str] = 'utf-8'
+    max_failures: int = 3
+    retry_delay: int = 10
+    validate_output: bool = False
+    message_context: Optional[str] = None
+    generate_gif: bool | str = False
+    available_file_paths: Optional[list[str]] = None
+    override_system_message: Optional[str] = None
+    extend_system_message: Optional[str] = None
+    tool_calling_method: Optional[ToolCallingMethod] = 'auto'
+    max_llm_json_retries: int = 3
+    save_file_path: str = "browser_agent_history.json"

chapter9/gaia-experience/AWorld/examples/browser_use/prompts.py

@@ -0,0 +1,212 @@
+# coding: utf-8
+
+from datetime import datetime
+from typing import List, Optional
+
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from examples.browser_use.common import AgentStepInfo
+from aworld.core.common import Observation, ActionResult
+
+PROMPT_TEMPLATE = """
+You are an AI agent designed to automate browser tasks. Your goal is to accomplish the ultimate task following the rules.
+
+# Input Format
+Task
+Previous steps
+Current URL
+Open Tabs
+Interactive Elements
+[index]<type>text</type>
+- index: Numeric identifier for interaction
+- type: HTML element type (button, input, etc.)
+- text: Element description
+Example:
+[33]<button>Submit Form</button>
+
+- Only elements with numeric indexes in [] are interactive
+- elements without [] provide only context
+
+# Response Rules
+1. RESPONSE FORMAT: You must ALWAYS respond with valid JSON in this exact format:
+{{"current_state": {{"evaluation_previous_goal": "Success|Failed|Unknown - Analyze the current elements and the image to check if the previous goals/actions are successful like intended by the task. Mention if something unexpected happened. Shortly state why/why not",
+"memory": "Description of what has been done and what you need to remember. Be very specific. Count here ALWAYS how many times you have done something and how many remain. E.g. 0 out of 10 websites analyzed. Continue with abc and xyz",
+"thought": "Your thought or reasoning based on the ultimate task and current observations",
+"next_goal": "What needs to be done with the next immediate action"}},
+"action":[{{"one_action_name": {{// action-specific parameter}}}}, // ... more actions in sequence]}}
+
+2. ACTIONS: You can specify multiple actions in the list to be executed in sequence. But always specify only one action name per item. Use maximum {max_actions} actions per sequence.
+Common action sequences:
+- Form filling: [{{"input_text": {{"index": 1, "text": "username"}}}}, {{"input_text": {{"index": 2, "text": "password"}}}}, {{"click_element": {{"index": 3}}}}]
+- Navigation and extraction: [{{"go_to_url": {{"url": "https://example.com"}}}}, {{"extract_content": {{"goal": "extract the names"}}}}]
+- Actions are executed in the given order
+- If the page changes after an action, the sequence is interrupted and you get the new state.
+- Only provide the action sequence until an action which changes the page state significantly.
+- Try to be efficient, e.g. fill forms at once, or chain actions where nothing changes on the page
+- only use multiple actions if it makes sense.
+
+3. ELEMENT INTERACTION:
+- Only use indexes of the interactive elements
+- Elements marked with "[]Non-interactive text" are non-interactive
+
+4. NAVIGATION & ERROR HANDLING:
+- If no suitable elements exist, use other functions to complete the task
+- If stuck, try alternative approaches - like going back to a previous page, new search, new tab etc.
+- Handle popups/cookies by accepting or closing them
+- Use scroll to find elements you are looking for
+- If you want to research something, open a new tab instead of using the current tab
+- If captcha pops up, try to solve it - else try a different approach
+- If the page is not fully loaded, use wait action
+
+5. TASK COMPLETION:
+- Use the done action as the last action as soon as the ultimate task is complete
+- Dont use "done" before you are done with everything the user asked you, except you reach the last step of max_steps. 
+- If you reach your last step, use the done action even if the task is not fully finished. Provide all the information you have gathered so far. If the ultimate task is completly finished set success to true. If not everything the user asked for is completed set success in done to false!
+- If you have to do something repeatedly for example the task says for "each", or "for all", or "x times", count always inside "memory" how many times you have done it and how many remain. Don't stop until you have completed like the task asked you. Only call done after the last step.
+- Don't hallucinate actions
+- Make sure you include everything you found out for the ultimate task in the done text parameter. Do not just say you are done, but include the requested information of the task. 
+
+6. VISUAL CONTEXT:
+- When an image is provided, use it to understand the page layout
+- Bounding boxes with labels on their top right corner correspond to element indexes
+
+7. Form filling:
+- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
+
+8. Long tasks:
+- Keep track of the status and subresults in the memory. 
+
+9. Extraction:
+- If your task is to find information - call extract_content on the specific pages to get and store the information.
+Your responses must be always JSON with the specified format. 
+"""
+
+
+class SystemPrompt:
+    def __init__(self,
+                 max_actions_per_step: int = 10,
+                 override_system_message: Optional[str] = None,
+                 extend_system_message: Optional[str] = None):
+        self.max_actions_per_step = max_actions_per_step
+        if override_system_message:
+            prompt = override_system_message
+        else:
+            prompt = PROMPT_TEMPLATE.format(max_actions=self.max_actions_per_step)
+
+        if extend_system_message:
+            prompt += f'\n{extend_system_message}'
+
+        self.system_message = SystemMessage(content=prompt)
+
+    def get_system_message(self) -> SystemMessage:
+        """
+        Get the system prompt for the agent.
+
+        Returns:
+            SystemMessage: Formatted system prompt
+        """
+        return self.system_message
+
+
+class AgentMessagePrompt:
+    def __init__(
+            self,
+            state: Observation,
+            result: Optional[List[ActionResult]] = None,
+            include_attributes: list[str] = [],
+            step_info: Optional[AgentStepInfo] = None,
+    ):
+        self.state = state
+        self.result = result
+        self.include_attributes = include_attributes
+        self.step_info = step_info
+
+    def get_user_message(self, use_vision: bool = True) -> HumanMessage:
+        elements_text = self.state.dom_tree.element_tree.clickable_elements_to_string(
+            include_attributes=self.include_attributes)
+
+        pixels_above = self.state.info.get('pixels_above', 0)
+        pixels_below = self.state.info.get('pixels_below', 0)
+
+        if elements_text != '':
+            if pixels_above > 0:
+                elements_text = (
+                    f'... {pixels_above} pixels above - scroll or extract content to see more ...\n{elements_text}'
+                )
+            else:
+                elements_text = f'[Start of page]\n{elements_text}'
+            if pixels_below > 0:
+                elements_text = (
+                    f'{elements_text}\n... {pixels_below} pixels below - scroll or extract content to see more ...'
+                )
+            else:
+                elements_text = f'{elements_text}\n[End of page]'
+        else:
+            elements_text = 'empty page'
+
+        if self.step_info:
+            step_info_description = f'Current step: {self.step_info.number}/{self.step_info.max_steps}'
+        else:
+            step_info_description = ''
+        time_str = datetime.now().strftime('%Y-%m-%d %H:%M')
+        step_info_description += f'Current date and time: {time_str}'
+
+        state_description = f"""
+[Task history memory ends]
+[Current state starts here]
+The following is one-time information - if you need to remember it write it to memory:
+Current url: {self.state.info.get("url")}
+Interactive elements from top layer of the current page inside the viewport:
+{elements_text}
+{step_info_description}
+"""
+
+        if self.result:
+            for i, result in enumerate(self.result):
+                if result.content:
+                    state_description += f'\nAction result {i + 1}/{len(self.result)}: {result.content}'
+                if result.error:
+                    # only use last line of error
+                    error = result.error.split('\n')[-1]
+                    state_description += f'\nAction error {i + 1}/{len(self.result)}: ...{error}'
+
+        if self.state.image and use_vision == True:
+            # Format message for vision model
+            return HumanMessage(
+                content=[
+                    {'type': 'text', 'text': state_description},
+                    {
+                        'type': 'image_url',
+                        'image_url': {'url': f'data:image/png;base64,{self.state.image}'},  # , 'detail': 'low'
+                    },
+                ]
+            )
+
+        return HumanMessage(content=state_description)
+
+
+class PlannerPrompt(SystemPrompt):
+    def get_system_message(self) -> SystemMessage:
+        return SystemMessage(
+            content="""You are a planning agent that helps break down tasks into smaller steps and reason about the current state.
+Your role is to:
+1. Analyze the current state and history
+2. Evaluate progress towards the ultimate goal
+3. Identify potential challenges or roadblocks
+4. Suggest the next high-level steps to take
+
+Inside your messages, there will be AI messages from different agents with different formats.
+
+Your output format should be always a JSON object with the following fields:
+{
+    "state_analysis": "Brief analysis of the current state and what has been done so far",
+    "progress_evaluation": "Evaluation of progress towards the ultimate goal (as percentage and description)",
+    "challenges": "List any potential challenges or roadblocks",
+    "next_steps": "List 2-3 concrete next steps to take",
+    "reasoning": "Explain your reasoning for the suggested next steps"
+}
+
+Ignore the other AI messages output structures.
+don't forget the index param for input_text action.
+Keep your responses concise and focused on actionable insights."""
+        )

chapter9/gaia-experience/AWorld/examples/browser_use/requirements.txt

@@ -0,0 +1,6 @@
+langchain~=0.3.20
+langchain-openai~=0.3.8
+langchain-ollama~=0.2.3
+langchain-anthropic~=0.3.9
+langchain-mistralai~=0.2.7
+langchain-google-genai~=2.1.0

chapter9/gaia-experience/AWorld/examples/browser_use/run.py

@@ -0,0 +1,56 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+import os
+
+from aworld.config.conf import ModelConfig
+from aworld.core.task import Task
+from aworld.runner import Runners
+from examples.browser_use.agent import BrowserAgent
+from examples.browser_use.config import BrowserAgentConfig
+from examples.common.tools.common import Agents, Tools
+from examples.common.tools.conf import BrowserToolConfig
+
+# os.environ["LLM_MODEL_NAME"] = "YOUR_LLM_MODEL_NAME"
+# os.environ["LLM_BASE_URL"] = "YOUR_LLM_BASE_URL"
+# os.environ["LLM_API_KEY"] = "YOUR_LLM_API_KEY"
+
+if __name__ == '__main__':
+    llm_config = ModelConfig(
+        llm_provider=os.getenv("LLM_PROVIDER", "openai"),
+        llm_model_name=os.getenv("LLM_MODEL_NAME"),
+        llm_base_url=os.getenv("LLM_BASE_URL"),
+        llm_api_key=os.getenv("LLM_API_KEY"),
+        llm_temperature=os.getenv("LLM_TEMPERATURE", 0.0)
+    )
+    browser_tool_config = BrowserToolConfig(width=1280,
+                                            height=720,
+                                            headless=False,
+                                            keep_browser_open=True,
+                                            use_async=True,
+                                            custom_executor=True,
+                                            llm_config=llm_config)
+    agent_config = BrowserAgentConfig(
+        name=Agents.BROWSER.value,
+        tool_calling_method="raw",
+        llm_config=llm_config,
+        max_actions_per_step=10,
+        max_input_tokens=128000,
+        working_dir="",
+        # llm model not supported vision, need to set `False`
+        # use_vision=False
+    )
+
+    task_config = {
+        'max_steps': 100,
+        'max_actions_per_step': 100
+    }
+
+    task = Task(
+        input="""step1: first go to https://www.dangdang.com/ and search for 'the little prince' and rank by sales from high to low, get the first 5 results and put the products info in memory.
+        step 2: write each product's title, price, discount, and publisher information to a fully structured HTML document with write_to_file, ensuring that the data is presented in a table with visible grid lines.
+        step3: open the html file in browser by go_to_url""",
+        agent=BrowserAgent(conf=agent_config, name=Agents.BROWSER.value, tool_names=[Tools.BROWSER.name]),
+        tools_conf={Tools.BROWSER.value: browser_tool_config},
+        conf=task_config
+    )
+    Runners.sync_run_task(task)

chapter9/gaia-experience/AWorld/examples/browser_use/utils.py

@@ -0,0 +1,199 @@
+# coding: utf-8
+import requests
+import json
+from io import BytesIO
+import os
+from typing import Any, Optional, Type
+import base64
+
+from langchain_core.messages import (
+    AIMessage,
+    BaseMessage,
+    HumanMessage,
+    SystemMessage,
+    ToolMessage,
+)
+
+from aworld.logs.util import logger
+
+
+def extract_json_from_model_output(content: str) -> dict:
+    """Extract JSON from model output, handling both plain JSON and code-block-wrapped JSON."""
+    try:
+        # If content is wrapped in code blocks, extract just the JSON part
+        if '```' in content:
+            # Find the JSON content between code blocks
+            content = content.split('```')[1]
+            # Remove language identifier if present (e.g., 'json\n')
+            if '\n' in content:
+                content = content.split('\n', 1)[1]
+        # Parse the cleaned content
+        return json.loads(content)
+    except json.JSONDecodeError as e:
+        logger.warning(f'Failed to parse model output: {content} {str(e)}')
+        raise ValueError('Could not parse response.')
+
+
+def convert_input_messages(input_messages: list[BaseMessage], model_name: Optional[str]) -> list[BaseMessage]:
+    """Convert input messages to a format that is compatible with the planner model"""
+    if model_name is None:
+        return input_messages
+    if model_name == 'deepseek-reasoner' or model_name.startswith('deepseek-r1'):
+        converted_input_messages = _convert_messages_for_non_function_calling_models(input_messages)
+        merged_input_messages = _merge_successive_messages(converted_input_messages, HumanMessage)
+        merged_input_messages = _merge_successive_messages(merged_input_messages, AIMessage)
+        return merged_input_messages
+    return input_messages
+
+
+def _convert_messages_for_non_function_calling_models(input_messages: list[BaseMessage]) -> list[BaseMessage]:
+    """Convert messages for non-function-calling models"""
+    output_messages = []
+    for message in input_messages:
+        if isinstance(message, HumanMessage):
+            output_messages.append(message)
+        elif isinstance(message, SystemMessage):
+            output_messages.append(message)
+        elif isinstance(message, ToolMessage):
+            output_messages.append(HumanMessage(content=message.content))
+        elif isinstance(message, AIMessage):
+            # check if tool_calls is a valid JSON object
+            if message.tool_calls:
+                tool_calls = json.dumps(message.tool_calls)
+                output_messages.append(AIMessage(content=tool_calls))
+            else:
+                output_messages.append(message)
+        else:
+            raise ValueError(f'Unknown message type: {type(message)}')
+    return output_messages
+
+
+def _merge_successive_messages(messages: list[BaseMessage], class_to_merge: Type[BaseMessage]) -> list[BaseMessage]:
+    """Some models like deepseek-reasoner dont allow multiple human messages in a row. This function merges them into one."""
+    merged_messages = []
+    streak = 0
+    for message in messages:
+        if isinstance(message, class_to_merge):
+            streak += 1
+            if streak > 1:
+                if isinstance(message.content, list):
+                    merged_messages[-1].content += message.content[0]['text']  # type:ignore
+                else:
+                    merged_messages[-1].content += message.content
+            else:
+                merged_messages.append(message)
+        else:
+            merged_messages.append(message)
+            streak = 0
+    return merged_messages
+
+
+def save_conversation(input_messages: list[BaseMessage], response: Any, target: str,
+                      encoding: Optional[str] = None) -> None:
+    """Save conversation history to file."""
+
+    # create folders if not exists
+    os.makedirs(os.path.dirname(target), exist_ok=True)
+
+    with open(
+            target,
+            'w',
+            encoding=encoding,
+    ) as f:
+        _write_messages_to_file(f, input_messages)
+        _write_response_to_file(f, response)
+
+
+def _write_messages_to_file(f: Any, messages: list[BaseMessage]) -> None:
+    """Write messages to conversation file"""
+    for message in messages:
+        f.write(f' {message.__class__.__name__} \n')
+
+        if isinstance(message.content, list):
+            for item in message.content:
+                if isinstance(item, dict) and item.get('type') == 'text':
+                    f.write(item['text'].strip() + '\n')
+        elif isinstance(message.content, str):
+            try:
+                content = json.loads(message.content)
+                f.write(json.dumps(content, indent=2) + '\n')
+            except json.JSONDecodeError:
+                f.write(message.content.strip() + '\n')
+
+        f.write('\n')
+
+
+def _write_response_to_file(f: Any, response: Any) -> None:
+    """Write model response to conversation file"""
+    f.write(' RESPONSE\n')
+    f.write(json.dumps(json.loads(response.model_dump_json(exclude_unset=True)), indent=2))
+
+
+# Add token counting related functions
+# Note: These functions have been moved from memory.py and agent.py to utils.py, removing the dependency on MessageManager class
+
+def estimate_text_tokens(text: str, estimated_characters_per_token: int = 3) -> int:
+    """Roughly estimate token count in text
+    
+    Args:
+        text: The text to estimate tokens for
+        estimated_characters_per_token: Estimated characters per token, default is 3
+        
+    Returns:
+        Estimated token count
+    """
+    if not text:
+        return 0
+    # Use character count divided by average characters per token to estimate tokens
+    return len(text) // estimated_characters_per_token
+
+
+def estimate_message_tokens(message: BaseMessage, image_tokens: int = 800, 
+                       estimated_characters_per_token: int = 3) -> int:
+    """Roughly estimate token count for a single message
+    
+    Args:
+        message: The message to estimate tokens for
+        image_tokens: Estimated tokens per image, default is 800
+        estimated_characters_per_token: Estimated characters per token, default is 3
+        
+    Returns:
+        Estimated token count
+    """
+    tokens = 0
+    # Handle tuple case
+    if isinstance(message, tuple):
+        # Convert to string and estimate tokens
+        message_str = str(message)
+        return estimate_text_tokens(message_str, estimated_characters_per_token)
+        
+    if isinstance(message.content, list):
+        for item in message.content:
+            if 'image_url' in item:
+                tokens += image_tokens
+            elif isinstance(item, dict) and 'text' in item:
+                tokens += estimate_text_tokens(item['text'], estimated_characters_per_token)
+    else:
+        msg = message.content
+        if hasattr(message, 'tool_calls'):
+            msg += str(message.tool_calls)  # type: ignore
+        tokens += estimate_text_tokens(msg, estimated_characters_per_token)
+    return tokens
+
+
+def estimate_messages_tokens(messages: list[BaseMessage], image_tokens: int = 800,
+                        estimated_characters_per_token: int = 3) -> int:
+    """Roughly estimate total token count for a list of messages
+    
+    Args:
+        messages: The list of messages to estimate tokens for
+        image_tokens: Estimated tokens per image, default is 800
+        estimated_characters_per_token: Estimated characters per token, default is 3
+        
+    Returns:
+        Estimated total token count
+    """
+    total_tokens = 0
+    for msg in messages:
+        total_tokens += estimate_message_tokens(msg, image_tokens, estimated_characters_per_token)
+    return total_tokens