ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter9/gaia-experience/AWorld/examples/common/README.md

@@ -0,0 +1,3 @@
+# Common module
+
+Tools commonly used in examples.

chapter9/gaia-experience/AWorld/examples/common/tools/__init__.py

@@ -0,0 +1,9 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+from aworld.core.tool.base import Tool, AsyncTool
+from aworld.core.tool.action import ExecutableAction
+from aworld.utils.common import scan_packages
+
+scan_packages("examples.common.tools", [Tool, AsyncTool, ExecutableAction])
+from examples.common.tools.browsers.action.actions import *

chapter9/gaia-experience/AWorld/examples/common/tools/android/README.md

@@ -0,0 +1,79 @@
+## Android Environment Setup Guide
+
+This guide will help you set up a local Android environment for AgentWorld.
+
+### Installation Steps
+
+1. **Download and Install Android Studio**
+   - Visit [https://developer.android.com/studio](https://developer.android.com/studio)
+   - Download and install the latest version for your operating system
+
+2. **Install ADB and Android Emulator**
+   - Open Android Studio
+   - Click on the top menu: Tools → SDK Manager
+        <img src="../../../readme_assets/android_step1.png" width="70%" alt="SDK Manager">
+       <!-- ![Agent World Framework](../../readme_assets/android_step1.png){:style="width:200px; height:auto;"} -->
+   - Check the following components:
+     - Android SDK Build-Tools
+     - Android SDK Command-line Tools
+     - Android Emulator
+     - Android SDK Platform-Tools
+   - Click "Apply" to install these components
+      <img src="../../../readme_assets/android_step2.png" width="70%" alt="Check components">
+   - **Important**: Copy the installation directory path (you'll need it later for configuration)
+
+3. **Create a Virtual Device**
+   - From the main menu, select: View → Tool Windows → Device Manager
+    <img src="../../../readme_assets/android_step3.png" width="70%" alt="Device Manager">
+   - Click the "+" button, then "Create Virtual Device"
+   <img src="../../../readme_assets/android_step4.png" width="70%" alt="button">
+   - Select a device (e.g., Medium Phone), then click "Next"
+   <img src="../../../readme_assets/android_step5.png" width="70%" alt="next">
+   - Select a image (e.g., VanillalceCream), then click "Next"
+   <img src="../../../readme_assets/android_step6.png" width="70%" alt="next">
+   - Configure device settings as needed, then click "Finish"
+   - **Important**: Note down the AVD ID (device name) for later use
+   <img src="../../../readme_assets/android_step7.png" width="70%" alt="avd id">
+
+4. **Configure in Your Code**
+   - Method 1: Default Acquisition of Emulator and ADB Installation Paths
+     - Only set the AVD_ID copied during the earlier installation process.
+   - Method 2: Manually Specify Emulator and ADB Installation Paths.Provide the following:
+     - AVD_ID: The name of the virtual device you created
+     - ADB path: Your SDK directory + "/platform-tools/adb"
+     - Emulator path: Your SDK directory + "/emulator/emulator"
+### Example Code
+#### Method 1
+
+```python
+from examples.common.tools.android.action.adb_controller import ADBController
+
+# Initialize the Android controller
+android_controller = ADBController(avd_name="Medium_Phone_API_35")
+```
+#### Method 2
+
+```python
+from examples.common.tools.android.action.adb_controller import ADBController
+
+# Initialize the Android controller
+android_controller = ADBController(
+    avd_name="Medium_Phone_API_35",
+    adb_path="/Users/username/Library/Android/sdk/platform-tools/adb",
+    emulator_path="/Users/username/Library/Android/sdk/emulator/emulator"
+)
+
+# Now you can use this controller with your agent
+```
+
+### Troubleshooting
+
+- If the emulator fails to start, try increasing the memory allocation in the AVD settings
+- Make sure your paths are correct for your operating system:
+  - Windows: Use backslashes or raw strings (r"C:\path\to\sdk")
+  - macOS/Linux: Use forward slashes as shown in the example
+
+### Additional Resources
+
+- [Android SDK Official Documentation](https://developer.android.com/studio/intro)
+- [Android Emulator Documentation](https://developer.android.com/studio/run/emulator)

chapter9/gaia-experience/AWorld/examples/common/tools/android/__init__.py

@@ -0,0 +1,2 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.

chapter9/gaia-experience/AWorld/examples/common/tools/android/action/actions.py

@@ -0,0 +1,77 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+import json
+
+from examples.common.tools.tool_action import AndroidAction
+from aworld.core.tool.action_factory import ActionFactory
+from aworld.core.common import ActionModel, ActionResult
+from examples.common.tools.android.action.adb_controller import ADBController
+from examples.common.tools.android.config.android_action_space import AndroidActionParamEnum
+from aworld.core.tool.action import ExecutableAction
+
+
+@ActionFactory.register(name=AndroidAction.TAP.value.name,
+                        desc=AndroidAction.TAP.value.desc,
+                        tool_name="android")
+class Tap(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> ActionResult:
+        controller: ADBController = kwargs.get('controller')
+        tap_index = action.params[AndroidActionParamEnum.TAP_INDEX.value]
+        if tap_index is None:
+            raise Exception(f'Invalid action: {action}')
+        controller.tap(tap_index)
+        return ActionResult(content="", keep=True)
+
+
+@ActionFactory.register(name=AndroidAction.INPUT_TEXT.value.name,
+                        desc=AndroidAction.INPUT_TEXT.value.desc,
+                        tool_name="android")
+class InputText(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> ActionResult:
+        controller: ADBController = kwargs.get('controller')
+        input_text = action.params[AndroidActionParamEnum.INPUT_TEXT.value]
+        if input_text is None:
+            raise Exception(f'Invalid action: {action}')
+        controller.text(input_text)
+        return ActionResult(content="", keep=True)
+
+
+@ActionFactory.register(name=AndroidAction.LONG_PRESS.value.name,
+                        desc=AndroidAction.LONG_PRESS.value.desc,
+                        tool_name="android")
+class LongPress(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> ActionResult:
+        controller: ADBController = kwargs.get('controller')
+        long_press_index = action.params[AndroidActionParamEnum.LONG_PRESS_INDEX.value]
+        if long_press_index is None:
+            raise Exception(f'Invalid action: {action}')
+        controller.long_press(long_press_index)
+        return ActionResult(content="", keep=True)
+
+
+@ActionFactory.register(name=AndroidAction.SWIPE.value.name,
+                        desc=AndroidAction.SWIPE.value.desc,
+                        tool_name="android")
+class Swipe(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> ActionResult:
+        controller: ADBController = kwargs.get('controller')
+        swipe_start_index = action.params[AndroidActionParamEnum.SWIPE_START_INDEX.value]
+        direction = action.params[AndroidActionParamEnum.DIRECTION.value]
+        dist = action.params.get(AndroidActionParamEnum.DIST.value, None)
+        if swipe_start_index is None or direction is None:
+            raise Exception(f'Invalid action: {action}')
+        if dist:
+            controller.swipe(swipe_start_index, direction, dist)
+        else:
+            controller.swipe(swipe_start_index, direction)
+        return ActionResult(content="", keep=True)
+
+
+@ActionFactory.register(name=AndroidAction.DONE.value.name,
+                        desc=AndroidAction.DONE.value.desc,
+                        tool_name="android")
+class Done(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> ActionResult:
+        output_dict = action.model_dump(exclude={'success'})
+        return ActionResult(is_done=True, success=True, content=json.dumps(output_dict))

chapter9/gaia-experience/AWorld/examples/common/tools/android/action/adb_controller.py

@@ -0,0 +1,541 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+import subprocess
+import time
+import re
+import traceback
+from time import sleep
+from typing import Optional, Tuple, List
+import base64
+
+import xml.etree.ElementTree as ET
+import os
+
+from aworld.logs.util import logger, color_log, Color
+from aworld.utils import import_package
+
+configs = {"MIN_DIST": 30}
+
+
+class AndroidElement:
+    def __init__(self, uid, bbox, attrib):
+        self.uid = uid
+        self.bbox = bbox
+        self.attrib = attrib
+        import_package('cv2', install_name='opencv-python')
+        import_package('pyshine')
+
+def get_id_from_element(elem):
+    bounds = elem.attrib["bounds"][1:-1].split("][")
+    x1, y1 = map(int, bounds[0].split(","))
+    x2, y2 = map(int, bounds[1].split(","))
+    elem_w, elem_h = x2 - x1, y2 - y1
+    if "resource-id" in elem.attrib and elem.attrib["resource-id"]:
+        elem_id = elem.attrib["resource-id"].replace(":", ".").replace("/", "_")
+    else:
+        elem_id = f"{elem.attrib['class']}_{elem_w}_{elem_h}"
+    if "content-desc" in elem.attrib and elem.attrib["content-desc"] and len(elem.attrib["content-desc"]) < 20:
+        content_desc = elem.attrib['content-desc'].replace("/", "_").replace(" ", "").replace(":", "_")
+        elem_id += f"_{content_desc}"
+    return elem_id
+
+
+def traverse_tree(xml_path, elem_list, attrib, add_index=False):
+    path = []
+    for event, elem in ET.iterparse(xml_path, ['start', 'end']):
+        if event == 'start':
+            path.append(elem)
+            if attrib in elem.attrib and elem.attrib[attrib] == "true":
+                parent_prefix = ""
+                if len(path) > 1:
+                    parent_elem = path[-2]
+                    # Checks if the parent element has the required attributes
+                    has_bounds = "bounds" in parent_elem.attrib
+                    has_rid_or_class = "resource-id" in parent_elem.attrib or "class" in parent_elem.attrib
+                    if has_bounds and has_rid_or_class:
+                        parent_prefix = get_id_from_element(parent_elem)
+                bounds = elem.attrib["bounds"][1:-1].split("][")
+                x1, y1 = map(int, bounds[0].split(","))
+                x2, y2 = map(int, bounds[1].split(","))
+                center = (x1 + x2) // 2, (y1 + y2) // 2
+                elem_id = get_id_from_element(elem)
+                if parent_prefix:
+                    elem_id = parent_prefix + "_" + elem_id
+                if add_index:
+                    elem_id += f"_{elem.attrib['index']}"
+                close = False
+                for e in elem_list:
+                    bbox = e.bbox
+                    center_ = (bbox[0][0] + bbox[1][0]) // 2, (bbox[0][1] + bbox[1][1]) // 2
+                    dist = (abs(center[0] - center_[0]) ** 2 + abs(center[1] - center_[1]) ** 2) ** 0.5
+                    if dist <= configs["MIN_DIST"]:
+                        close = True
+                        break
+                if not close:
+                    elem_list.append(AndroidElement(elem_id, ((x1, y1), (x2, y2)), attrib))
+
+        if event == 'end':
+            path.pop()
+
+
+def create_directory_for_file(file_path):
+    # Extract the directory from the file path
+    directory = os.path.dirname(file_path)
+
+    # Check if the directory exists
+    if not os.path.exists(directory):
+        # Create the directory
+        os.makedirs(directory)
+        # Print the absolute path of the directory
+        absolute_directory_path = os.path.abspath(directory)
+        logger.info(f"Directory absolute path: {absolute_directory_path}")
+
+
+def draw_bbox_multi(img_path, output_path, elem_list):
+    import cv2
+    import pyshine as ps
+
+    imgcv = cv2.imread(img_path)
+    count = 1
+    for elem in elem_list:
+        try:
+            top_left = elem.bbox[0]
+            bottom_right = elem.bbox[1]
+            left, top = top_left[0], top_left[1]
+            right, bottom = bottom_right[0], bottom_right[1]
+
+            # draw rectangle
+            cv2.rectangle(imgcv,
+                          (left, top),
+                          (right, bottom),
+                          (0, 0, 221),
+                          3)
+
+            label = str(count)
+            imgcv = ps.putBText(imgcv, label, text_offset_x=(left + right) // 2 + 10,
+                                text_offset_y=(top + bottom) // 2 + 10,
+                                vspace=10, hspace=10, font_scale=1, thickness=2, background_RGB=(221, 0, 0),
+                                text_RGB=(255, 255, 255), alpha=0.0)
+
+        except Exception as e:
+            color_log(f"ERROR: An exception occurs while labeling the image\n{e}", Color.red)
+            logger.info(traceback.print_exc())
+        count += 1
+    cv2.imwrite(output_path, imgcv)
+    return imgcv
+
+
+def draw_grid(img_path, output_path):
+    import cv2
+
+    def get_unit_len(n):
+        for i in range(1, n + 1):
+            if n % i == 0 and 120 <= i <= 180:
+                return i
+        return -1
+
+    image = cv2.imread(img_path)
+    height, width, _ = image.shape
+    color = (255, 116, 113)
+    unit_height = get_unit_len(height)
+    if unit_height < 0:
+        unit_height = 120
+    unit_width = get_unit_len(width)
+    if unit_width < 0:
+        unit_width = 120
+    thick = int(unit_width // 50)
+    rows = height // unit_height
+    cols = width // unit_width
+    for i in range(rows):
+        for j in range(cols):
+            label = i * cols + j + 1
+            left = int(j * unit_width)
+            top = int(i * unit_height)
+            right = int((j + 1) * unit_width)
+            bottom = int((i + 1) * unit_height)
+            cv2.rectangle(image, (left, top), (right, bottom), color, thick // 2)
+            cv2.putText(image, str(label), (left + int(unit_width * 0.05) + 3, top + int(unit_height * 0.3) + 3), 0,
+                        int(0.01 * unit_width), (0, 0, 0), thick)
+            cv2.putText(image, str(label), (left + int(unit_width * 0.05), top + int(unit_height * 0.3)), 0,
+                        int(0.01 * unit_width), color, thick)
+    cv2.imwrite(output_path, image)
+    return rows, cols
+
+
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+
+
+class ADBController:
+    def __init__(self, avd_name: str = None,
+                 adb_path: str = os.path.expanduser('~') + "/Library/Android/sdk/platform-tools/adb",
+                 emulator_path: str = os.path.expanduser('~') + "/Library/Android/sdk/emulator/emulator",
+                 timeout: int = 30):
+        self.avd_name = avd_name
+        self.adb_path = adb_path
+        self.emulator_path = emulator_path
+        self.timeout = timeout
+        self.emulator_process = None
+        self.device_serial = "emulator-5554"  # default
+        self.current_elem_list = []
+        self.width, self.height = 0, 0
+
+    def start_emulator(self, avd_name: str = None, headless: bool = False,
+                       max_retry: int = 2) -> bool:
+        avd = avd_name or self.avd_name
+        if not avd:
+            raise ValueError("AVD name must be specified")
+
+        for attempt in range(max_retry + 1):
+            if self._start_emulator_process(avd, headless):
+                if self._wait_for_device():
+                    logger.info(f"start success，attempt count：{attempt + 1}")
+                    self.width, self.height = self.get_screen_size()
+                    return True
+                self.stop_emulator()
+        return False
+
+    def _start_emulator_process(self, avd: str, headless: bool) -> bool:
+        try:
+            cmd = [
+                self.emulator_path,
+                f"@{avd}",
+                "-no-snapshot",
+                "-no-audio",
+                "-gpu", "swiftshader",
+                "-wipe-data"
+            ]
+            if headless:
+                cmd.append("-no-window")
+
+            self.emulator_process = subprocess.Popen(
+                cmd,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.STDOUT
+            )
+            return True
+        except Exception as e:
+            logger.warning(f"adb start fail: {str(e)}")
+            return False
+
+    def stop_emulator(self) -> bool:
+        try:
+            result = subprocess.run(
+                [self.adb_path, "-s", self.device_serial, "emu", "kill"],
+                timeout=self.timeout,
+                capture_output=True,
+                text=True
+            )
+            return "OK" in result.stdout
+        except subprocess.TimeoutExpired:
+            return False
+        finally:
+            if self.emulator_process:
+                self.emulator_process.terminate()
+
+    def execute_adb(self, command: list, device_serial: str = None) -> Tuple[bool, str]:
+        """execute adb command"""
+        device = device_serial or self.device_serial
+        full_cmd = [self.adb_path, "-s", device] + command
+
+        try:
+            result = subprocess.run(
+                full_cmd,
+                timeout=self.timeout,
+                check=True,
+                capture_output=True,
+                text=True
+            )
+            return True, result.stdout.strip()
+        except subprocess.CalledProcessError as e:
+            return False, f"Command failed: {e.stderr}"
+        except Exception as e:
+            return False, str(e)
+
+    def execute_adb_with_stdout(self, command: List[str]) -> Tuple[bool, Optional[str]]:
+        try:
+            result = subprocess.run(
+                ["adb", "-s", self.device_serial] + command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                timeout=10
+            )
+            if result.returncode == 0:
+                return True, result.stdout.strip()
+            else:
+                return False, None
+        except subprocess.TimeoutExpired:
+            return False, None
+        except Exception as e:
+            return False, None
+
+    # ---------- device operate ----------
+
+    def screenshot(self, save_path: str) -> bool:
+        timestamp = int(time.time())
+        remote_path = f"/sdcard/screenshot_{timestamp}.png"
+
+        success, _ = self.execute_adb(["shell", "screencap", "-p", remote_path])
+        if not success:
+            return False
+
+        return self._pull_file(remote_path, save_path)
+
+    def dump_ui_xml(self, save_path: str) -> Optional[str]:
+        remote_path = "/sdcard/ui_dump.xml"
+        success, _ = self.execute_adb(["shell", "uiautomator", "dump", remote_path])
+        if not success:
+            logger.info("dump ui xml fail")
+            return None
+        success = self._pull_file(remote_path, save_path)
+        if not success:
+            logger.info("pull ui xml fail")
+            return None
+
+        with open(save_path, 'r', encoding='utf-8') as f:
+            xml_content = f.read()
+        return xml_content
+
+    def tap(self, element: int):
+        x, y = self.__get_element_center(element)
+        self.__tap_coordinate(x, y)
+
+    def text(self, text: str):
+        """
+        Input text, automatically replacing spaces with %s for proper ADB text input.
+
+        Parameters:
+            text: The text to input
+        """
+        # Replace spaces with %s for proper handling in ADB
+        formatted_text = text.replace(" ", "%s")
+        success, _ = self.execute_adb(["shell", "input", "text", formatted_text])
+        return success
+
+    def long_press(self, element: int):
+        x, y = self.__get_element_center(element)
+        self.__swipe_coordinate(x, y, x, y, 2000)
+
+    def swipe(self, element: int, direction: str, dist: str = "medium"):
+        """
+        Perform swipe operations based on screen element labels
+
+        Parameters：
+        element_tag: digital label displayed on the interface (1-based)
+        direction: swipe direction ["up", "down", "left", "right"]
+        dist: swipe distance ["short", "medium", "long"]
+        """
+
+        # 获取元素坐标
+        x, y = self.__get_element_center(element)
+
+        unit_dist = int(self.width / 10)
+        if dist == "long":
+            unit_dist *= 3
+        elif dist == "medium":
+            unit_dist *= 2
+        if direction == "up":
+            offset = 0, -2 * unit_dist
+        elif direction == "down":
+            offset = 0, 2 * unit_dist
+        elif direction == "left":
+            offset = -1 * unit_dist, 0
+        elif direction == "right":
+            offset = unit_dist, 0
+        else:
+            return False
+
+        self.__swipe_coordinate(x, y, x + offset[0], y + offset[1])
+
+    def screenshot_and_annotate(self, name_prefix=None, return_base64=True):
+        import cv2
+
+        """Collect screen information and mark interactive elements, and return data containing Base64 images"""
+        sleep(3)
+        if name_prefix is None:
+            name_prefix = str(time.time())
+        tmp_files_dir = os.path.join(os.path.dirname(__file__), "tmp_files")
+        os.makedirs(tmp_files_dir, exist_ok=True)
+        screenshot_path = os.path.join(tmp_files_dir, f"{name_prefix}_origin.png")
+        screenshot_res = self.screenshot(screenshot_path)
+        xml_path = os.path.join(tmp_files_dir, f"{name_prefix}.xml")
+        xml_res = self.dump_ui_xml(xml_path)
+        if screenshot_res == "ERROR" or xml_res is None:
+            logger.warning(f"Failed to take screenshot or read XML")
+            return None, None
+
+        # Parsing interactive elements
+        clickable_list = []
+        focusable_list = []
+        traverse_tree(xml_path, clickable_list, "clickable", True)
+        traverse_tree(xml_path, focusable_list, "focusable", True)
+
+        # Merge a list of duplicate elements
+        elem_list = clickable_list.copy()
+        for elem in focusable_list:
+            bbox = elem.bbox
+            center = (bbox[0][0] + bbox[1][0]) // 2, (bbox[0][1] + bbox[1][1]) // 2
+            if not any(
+                    ((center[0] - ((e.bbox[0][0] + e.bbox[1][0]) // 2)) ** 2 +
+                     (center[1] - ((e.bbox[0][1] + e.bbox[1][1]) // 2)) ** 2) ** 0.5 <= configs["MIN_DIST"]
+                    for e in clickable_list
+            ):
+                elem_list.append(elem)
+
+        # Generate annotated images
+        labeled_path = os.path.join(tmp_files_dir, f"{name_prefix}_labeled.png")
+        labeled_img = draw_bbox_multi(screenshot_path, labeled_path, elem_list)
+
+        # Show Image Window
+        # cv2.imshow("image", labeled_img)
+        # cv2.waitKey(0)
+        # cv2.destroyAllWindows()
+
+        # Base64 encoding
+        base64_str = None
+        if return_base64:
+            # Convert color space BGR->RGB
+            rgb_image = cv2.cvtColor(labeled_img, cv2.COLOR_BGR2RGB)
+            # Compress to JPEG format (with adjustable quality parameters)
+            success, buffer = cv2.imencode(".jpg", rgb_image, [int(cv2.IMWRITE_JPEG_QUALITY), 85])
+            if success:
+                base64_str = base64.b64encode(buffer).decode("utf-8")
+
+        self.current_elem_list = elem_list.copy()
+        logger.info(f"Current elem size{len(self.current_elem_list)}")
+        return xml_res, base64_str
+
+    def setup_connection(self) -> bool:
+        """Intelligent initialization device connection"""
+        # Prioritize physical equipment testing
+        if self.__connect_physical_device():
+            return True
+
+        # Try connecting to the simulator
+        if self.avd_name and self.start_emulator():
+            return True
+
+        raise ConnectionError("No available device found, please connect your phone or configure the simulator")
+
+    # ---------- Helper Methods ----------
+    def __connect_physical_device(self) -> bool:
+        """Connect an authorized USB device"""
+        devices = self.__get_authorized_devices()
+        if not devices:
+            return False
+
+        self.device = devices[0]
+        logger.info(f"Connected physical device: {self.device}")
+        self.device_serial = self.device
+        self.width, self.height = self.get_screen_size()
+        return True
+
+    def __get_authorized_devices(self) -> list:
+        """Get a list of authorized devices"""
+        success, output = self.execute_adb(["devices"])
+        if not success:
+            return []
+
+        return [
+            line.split("\t")[0]
+            for line in output.splitlines()
+            if "\tdevice" in line and "emulator" not in line
+        ]
+
+    def __tap_coordinate(self, x: int, y: int) -> bool:
+        """Click screen coordinates"""
+        success, _ = self.execute_adb(["shell", "input", "tap", str(x), str(y)])
+        return success
+
+    def __get_element_center(self, elem_idx: int) -> tuple:
+        """Calculate the coordinates of the center of the element"""
+        tl, br = self.current_elem_list[int(elem_idx) - 1].bbox
+        return (tl[0] + br[0]) // 2, (tl[1] + br[1]) // 2
+
+    def __swipe_coordinate(self, x1: int, y1: int, x2: int, y2: int, duration: int = 300) -> bool:
+        """Slide Operation"""
+        success, _ = self.execute_adb([
+            "shell", "input", "swipe",
+            str(x1), str(y1), str(x2), str(y2),
+            str(duration)
+        ])
+        return success
+
+    def _wait_for_device(self, timeout: int = 300) -> bool:
+        """Three-level waiting detection strategy"""
+        start_time = time.time()
+        stages = {
+            "adb_connected": False,
+            "boot_completed": False,
+            "services_ready": False
+        }
+
+        while time.time() - start_time < timeout:
+            # Step 1: Detect adb connection
+            if not stages["adb_connected"]:
+                _, devices = self.execute_adb(["devices"])
+                if self.device_serial in devices:
+                    stages["adb_connected"] = True
+
+            # Step 2: Detection system boot completed
+            if stages["adb_connected"] and not stages["boot_completed"]:
+                _, output = self.execute_adb([
+                    "shell", "getprop", "sys.boot_completed"
+                ])
+                if output.strip() == "1":
+                    stages["boot_completed"] = True
+
+            # Step 3: Detecting Graphics Service Readiness
+            if stages["boot_completed"] and not stages["services_ready"]:
+                _, output = self.execute_adb([
+                    "shell", "service check SurfaceFlinger"
+                ])
+                if "found" in output.lower():
+                    return True
+
+        return False
+
+    def _pull_file(self, remote: str, local: str) -> bool:
+        """Pull device files to local"""
+        create_directory_for_file(local)
+        success, _ = self.execute_adb(["pull", remote, local])
+        if success:
+            self.execute_adb(["shell", "rm", remote])  # 清理临时文件
+        return success
+
+    def get_screen_size(self) -> Optional[Tuple[int, int]]:
+        """Get screen resolution"""
+        success, output = self.execute_adb(["shell", "wm", "size"])
+        if not success:
+            return None
+
+        match = re.search(r"(\d+)x(\d+)", output)
+        if match:
+            return int(match.group(1)), int(match.group(2))
+        return None
+
+
+if __name__ == "__main__":
+    # Examples
+    controller = ADBController(avd_name="Medium_Phone_API_35")
+
+    # controller.stop_emulator()
+    if controller.setup_connection():
+        logger.info("Simulator started successfully")
+        width, height = controller.get_screen_size()
+        logger.info(f"Get the screen size{width},{height}")
+
+        # Take screenshots and annotate them
+        controller.screenshot_and_annotate()
+        controller.swipe(6, "up")
+
+        # controller.screenshot_and_annotate()
+        # controller.tap(6)
+        xml_txt, base64_txt = controller.screenshot_and_annotate()
+        logger.info(xml_txt)
+
+        # controller.stop_emulator()
+        logger.info("Close the simulator")

chapter9/gaia-experience/AWorld/examples/common/tools/android/action/executor.py

@@ -0,0 +1,42 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+from typing import List
+from aworld.core.tool.action_factory import ActionFactory
+from aworld.core.common import ActionModel, ActionResult
+from aworld.logs.util import logger
+from examples.common.tools.android.action.adb_controller import ADBController
+from aworld.core.tool.base import ToolActionExecutor
+
+
+class AndroidToolActionExecutor(ToolActionExecutor):
+
+    def __init__(self, controller: ADBController):
+        self.controller = controller
+
+    def execute_action(self, actions: List[ActionModel], **kwargs) -> list[ActionResult]:
+        """Execute the specified android action sequence by agent policy.
+
+        Args:
+            actions: Tool action sequence.
+
+        Returns:
+            Browser action result list.
+        """
+        action_results = []
+        for action in actions:
+            action_result = self._exec(action, **kwargs)
+            action_results.append(action_result)
+        return action_results
+
+    def _exec(self, action_model: ActionModel, **kwargs):
+        action_name = action_model.action_name
+        if action_name not in ActionFactory:
+            action_name = action_model.tool_name + action_model.action_name
+            if action_name not in ActionFactory:
+                raise ValueError(f'Action {action_name} not found')
+
+        action = ActionFactory(action_name)
+        action_result = action.act(action_model, controller=self.controller, **kwargs)
+        logger.info(f"{action_name} execute finished")
+        return action_result

chapter9/gaia-experience/AWorld/examples/common/tools/android/android.py

@@ -0,0 +1,92 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+import traceback
+from pathlib import Path
+from typing import Any, Tuple, List, Dict
+
+from examples.common.tools.tool_action import AndroidAction
+from aworld.core.common import ActionModel, Observation, ActionResult
+from aworld.logs.util import logger
+from examples.common.tools.android.action.adb_controller import ADBController
+from examples.common.tools.android.action.executor import AndroidToolActionExecutor
+from examples.common.tools.conf import AndroidToolConfig
+from aworld.core.tool.base import ToolFactory, Tool
+from aworld.tools.utils import build_observation
+
+ALL_UNICODE_CHARS = frozenset(chr(i) for i in range(0x10FFFF + 1))
+
+
+@ToolFactory.register(name="android",
+                      desc="android",
+                      supported_action=AndroidAction,
+                      conf_file_name=f'android_tool.yaml',
+                      dir=f"{Path(__file__).parent.absolute()}")
+class AndroidTool(Tool):
+
+    def __init__(self, conf: AndroidToolConfig, **kwargs):
+        super(AndroidTool, self).__init__(conf, **kwargs)
+        self.controller = ADBController(avd_name=self.conf.get('avd_name'),
+                                        adb_path=self.conf.get('adb_path'),
+                                        emulator_path=self.conf.get('emulator_path'))
+
+        if self.conf.get("custom_executor"):
+            self.action_executor = AndroidToolActionExecutor(self.controller)
+
+    def reset(self, *, seed: int | None = None, options: Dict[str, str] | None = None) -> Tuple[
+        Observation, Dict[str, Any]]:
+        # self.controller.stop_emulator()
+        # self.controller.start_emulator()
+        self.controller.setup_connection()
+        logger.info("start emulator successfully...")
+        # snapshot screen and annotate
+        xml, pic_base64 = self.get_observation()
+        action_result_list = [ActionResult(content='start', keep=True)]
+        return build_observation(observer=self.name(),
+                                 ability='',
+                                 dom_tree=xml,
+                                 image=pic_base64,
+                                 action_result=action_result_list), {}
+
+    def do_step(self, action_list: List[ActionModel], **kwargs) -> Tuple[
+        Observation, float, bool, bool, Dict[str, Any]]:
+
+        exec_state = 0
+        fail_error = ""
+        action_result_list = None
+        try:
+            action_result_list = self.action_executor.execute_action(action_list, **kwargs)
+            exec_state = 1
+        except Exception as e:
+            traceback.print_exc()
+            fail_error = str(e)
+
+        terminated = kwargs.get("terminated", False)
+        if action_result_list:
+            for action_result in action_result_list:
+                if action_result.is_done:
+                    terminated = action_result.is_done
+                    self._finish = True
+
+        info = {"exception": fail_error}
+        info.update(kwargs)
+        xml, pic_base64 = self.get_observation()
+
+        return (build_observation(observer=self.name(),
+                                  ability=action_list[-1].action_name,
+                                  dom_tree=xml,
+                                  image=pic_base64,
+                                  action_result=action_result_list),
+                exec_state,
+                terminated,
+                kwargs.get("truncated", False),
+                info)
+
+    def close(self):
+        self.controller.stop_emulator()
+
+    def get_controller(self):
+        return self.controller
+
+    def get_observation(self) -> Observation:
+        return self.controller.screenshot_and_annotate()

chapter9/gaia-experience/AWorld/examples/common/tools/android/android_tool.yaml

@@ -0,0 +1,8 @@
+avd_name:
+adb_path:
+emulator_path:
+headless: False
+custom_executor: True
+enable_recording: False
+working_dir:
+max_retry: 3

chapter9/gaia-experience/AWorld/examples/common/tools/android/config/android_action_space.py

@@ -0,0 +1,26 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+from enum import Enum
+
+
+class AndroidActionParamEnum(Enum):
+    TAP_INDEX = "index"
+    LONG_PRESS_INDEX = "index"
+    INPUT_TEXT = "text"
+    SWIPE_START_INDEX = "index"
+    DIRECTION = "direction"
+    DIST = "dist"
+
+
+class DirectionParamEnum(Enum):
+    UP = "up"
+    DOWN = "down"
+    LEFT = "left"
+    RIGHT = "right"
+
+
+class DistParamEnum(Enum):
+    SHORT = "short"
+    MEDIUM = "medium"
+    LONG = "long"

chapter9/gaia-experience/AWorld/examples/common/tools/android/requirements.txt

@@ -0,0 +1,2 @@
+opencv-python~=4.11.0.86
+pyshine~=0.0.9

chapter9/gaia-experience/AWorld/examples/common/tools/apis/actions.py

@@ -0,0 +1,259 @@
+# coding: utf-8
+
+import json
+import os
+
+import requests
+
+from typing import Tuple, Any, List, Dict
+
+from examples.common.tools.tool_action import SearchAction
+from aworld.core.tool.action_factory import ActionFactory
+from aworld.core.common import ActionModel, ActionResult
+from aworld.logs.util import logger
+from aworld.utils import import_package
+from aworld.core.tool.action import ExecutableAction
+
+
+# @ActionFactory.register(name=SearchAction.WIKI.value.name,
+#                         desc=SearchAction.WIKI.value.desc,
+#                         tool_name='search_api')
+class SearchWiki(ExecutableAction):
+    def __init__(self):
+        import_package("wikipedia")
+
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        import wikipedia
+
+        query = action.params.get("query")
+        logger.info(f"Calling search_wiki api with query: {query}")
+
+        result: str = ''
+        try:
+            page = wikipedia.page(query)
+            result_dict = {
+                'url': page.url,
+                'title': page.title,
+                'content': page.content,
+            }
+            result = str(result_dict)
+        except wikipedia.exceptions.DisambiguationError as e:
+            result = wikipedia.summary(
+                e.options[0], sentences=5, auto_suggest=False
+            )
+        except wikipedia.exceptions.PageError:
+            result = (
+                "There is no page in Wikipedia corresponding to entity "
+                f"{query}, please specify another word to describe the"
+                " entity to be searched."
+            )
+        except Exception as e:
+            logger.error(f"An exception occurred during the search: {e}")
+            result = f"An exception occurred during the search: {e}"
+        logger.debug(f"wiki result: {result}")
+        return ActionResult(content=result, keep=True, is_done=True), None
+
+
+# @ActionFactory.register(name=SearchAction.DUCK_GO.value.name,
+#                         desc=SearchAction.DUCK_GO.value.desc,
+#                         tool_name="search_api")
+class Duckduckgo(ExecutableAction):
+    def __init__(self):
+        import_package("duckduckgo_search")
+
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        r"""Use DuckDuckGo search engine to search information for
+        the given query.
+
+        This function queries the DuckDuckGo API for related topics to
+        the given search term. The results are formatted into a list of
+        dictionaries, each representing a search result.
+
+        Args:
+            query (str): The query to be searched.
+            source (str): The type of information to query (e.g., "text",
+                "images", "videos"). Defaults to "text".
+            max_results (int): Max number of results, defaults to `5`.
+
+        Returns:
+            List[Dict[str, Any]]: A list of dictionaries where each dictionary
+                represents a search result.
+        """
+
+        from duckduckgo_search import DDGS
+
+        params = action.params
+        query = params.get("query")
+        max_results = params.get("max_results", 5)
+        source = params.get("source", "text")
+
+        logger.debug(f"Calling search_duckduckgo function with query: {query}")
+
+        ddgs = DDGS()
+        responses: List[Dict[str, Any]] = []
+
+        if source == "text":
+            try:
+                results = ddgs.text(keywords=query, max_results=max_results)
+            except Exception as e:
+                # Handle specific exceptions or general request exceptions
+                responses.append({"error": f"duckduckgo search failed.{e}"})
+                return ActionResult(content="duckduckgo search failed", keep=True), responses
+
+            for i, result in enumerate(results, start=1):
+                # Creating a response object with a similar structure
+                response = {
+                    "result_id": i,
+                    "title": result["title"],
+                    "description": result["body"],
+                    "url": result["href"],
+                }
+                responses.append(response)
+        elif source == "images":
+            try:
+                results = ddgs.images(keywords=query, max_results=max_results)
+            except Exception as e:
+                # Handle specific exceptions or general request exceptions
+                responses.append({"error": f"duckduckgo search failed.{e}"})
+                return ActionResult(content="duckduckgo search failed", keep=True), responses
+
+            # Iterate over results found
+            for i, result in enumerate(results, start=1):
+                # Creating a response object with a similar structure
+                response = {
+                    "result_id": i,
+                    "title": result["title"],
+                    "image": result["image"],
+                    "url": result["url"],
+                    "source": result["source"],
+                }
+                responses.append(response)
+        elif source == "videos":
+            try:
+                results = ddgs.videos(keywords=query, max_results=max_results)
+            except Exception as e:
+                # Handle specific exceptions or general request exceptions
+                responses.append({"error": f"duckduckgo search failed.{e}"})
+                return ActionResult(content="duckduckgo search failed", keep=True), responses
+
+            # Iterate over results found
+            for i, result in enumerate(results, start=1):
+                # Creating a response object with a similar structure
+                response = {
+                    "result_id": i,
+                    "title": result["title"],
+                    "description": result["description"],
+                    "embed_url": result["embed_url"],
+                    "publisher": result["publisher"],
+                    "duration": result["duration"],
+                    "published": result["published"],
+                }
+                responses.append(response)
+        logger.debug(f"Search results: {responses}")
+        return ActionResult(content=json.dumps(responses), keep=True, is_done=True), None
+
+
+# @ActionFactory.register(name=SearchAction.GOOGLE.value.name,
+#                         desc=SearchAction.GOOGLE.value.desc,
+#                         tool_name="search_api")
+class SearchGoogle(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        query = action.params.get("query")
+        num_result_pages = action.params.get("num_result_pages", 6)
+        # https://developers.google.com/custom-search/v1/overview
+        api_key = action.params.get("api_key", os.environ.get("GOOGLE_API_KEY"))
+        # https://cse.google.com/cse/all
+        engine_id = action.params.get("engine_id", os.environ.get("GOOGLE_ENGINE_ID"))
+        logger.debug(f"Calling search_google function with query: {query}")
+
+        # Using the first page
+        start_page_idx = 1
+        # Different language may get different result
+        search_language = "en"
+        # How many pages to return
+        num_result_pages = num_result_pages
+        # Constructing the URL
+        # Doc: https://developers.google.com/custom-search/v1/using_rest
+        url = f"https://www.googleapis.com/customsearch/v1?key={api_key}&cx={engine_id}&q={query}&start={start_page_idx}&lr={search_language}&num={num_result_pages}"
+        responses = []
+        try:
+            result = requests.get(url)
+            result.raise_for_status()
+            data = result.json()
+
+            # Get the result items
+            if "items" in data:
+                search_items = data.get("items")
+
+                for i, search_item in enumerate(search_items, start=1):
+                    # Check metatags are present
+                    if "pagemap" not in search_item:
+                        continue
+                    if "metatags" not in search_item["pagemap"]:
+                        continue
+                    if "og:description" in search_item["pagemap"]["metatags"][0]:
+                        long_description = search_item["pagemap"]["metatags"][0]["og:description"]
+                    else:
+                        long_description = "N/A"
+                    # Get the page title
+                    title = search_item.get("title")
+                    # Page snippet
+                    snippet = search_item.get("snippet")
+
+                    # Extract the page url
+                    link = search_item.get("link")
+                    response = {
+                        "result_id": i,
+                        "title": title,
+                        "description": snippet,
+                        "long_description": long_description,
+                        "url": link,
+                    }
+                    if "huggingface.co" in link:
+                        logger.warning(f"Filter out the link: {link}")
+                        continue
+                    responses.append(response)
+            else:
+                responses.append({"error": f"google search failed with response: {data}"})
+        except Exception as e:
+            logger.error(f"Google search failed with error: {e}")
+            responses.append({"error": f"google search failed with error: {e}"})
+
+        if len(responses) == 0:
+            responses.append(
+                "No relevant webpages found. Please simplify your query and expand the search space as much as you can, then try again.")
+        logger.debug(f"search result: {responses}")
+        responses.append(
+            "If the search result does not contain the information you want, please make reflection on your query: what went well, what didn't, then refine your search plan.")
+        return ActionResult(content=json.dumps(responses), keep=True, is_done=True), None
+
+
+@ActionFactory.register(name=SearchAction.BAIDU.value.name,
+                        desc=SearchAction.BAIDU.value.desc,
+                        tool_name="search_api")
+class SearchBaidu(ExecutableAction):
+    def __init__(self):
+        import_package("baidusearch")
+
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        from baidusearch.baidusearch import search
+
+        query = action.params.get("query")
+        num_results = action.params.get("num_results", 6)
+        num_results = int(num_results)
+        logger.debug(f"Calling search_baidu with query: {query}")
+
+        responses = []
+        try:
+            responses = search(query, num_results=num_results)
+        except Exception as e:
+            logger.error(f"Baidu search failed with error: {e}")
+            responses.append({"error": f"baidu search failed with error: {e}"})
+
+        if len(responses) == 0:
+            responses.append(
+                "No relevant webpages found. Please simplify your query and expand the search space as much as you can, then try again.")
+        logger.debug(f"search result: {responses}")
+        responses.append(
+            "If the search result does not contain the information you want, please make reflection on your query: what went well, what didn't, then refine your search plan.")
+        return ActionResult(content=json.dumps(responses), keep=True, is_done=True), None

chapter9/gaia-experience/AWorld/examples/common/tools/apis/search_api.py

@@ -0,0 +1,16 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+from pathlib import Path
+
+from aworld.core.tool.base import ToolFactory
+from aworld.tools.template_tool import TemplateTool
+from examples.common.tools.tool_action import SearchAction
+
+
+@ToolFactory.register(name="search_api",
+                      desc="search tool",
+                      supported_action=SearchAction,
+                      conf_file_name=f'search_api_tool.yaml',
+                      dir=f"{Path(__file__).parent.absolute()}")
+class SearchTool(TemplateTool):
+    """Search Tool"""

chapter9/gaia-experience/AWorld/examples/common/tools/apis/search_api_tool.yaml

@@ -0,0 +1,5 @@
+custom_executor: False
+enable_recording: False
+working_dir:
+max_retry: 3
+

chapter9/gaia-experience/AWorld/examples/common/tools/browsers/action/actions.py

@@ -0,0 +1,824 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+import os
+import traceback
+
+import asyncio
+import time
+from typing import Tuple, Any
+
+from examples.common.tools.tool_action import BrowserAction
+from aworld.core.tool.action_factory import ActionFactory
+from aworld.core.common import ActionModel, ActionResult, Observation
+from examples.common.tools.browsers.util.dom import DOMElementNode
+from aworld.logs.util import logger
+from examples.common.tools.browsers.action.utils import DomUtil
+from aworld.core.tool.action import ExecutableAction
+from aworld.utils import import_packages
+from aworld.models.llm import get_llm_model, call_llm_model
+
+
+def get_page(**kwargs):
+    tool = kwargs.get("tool")
+    if tool is None:
+        page = kwargs.get('page')
+    else:
+        page = tool.page
+    return page
+
+
+def get_browser(**kwargs):
+    tool = kwargs.get("tool")
+    if tool is None:
+        page = kwargs.get('browser')
+    else:
+        page = tool.context
+    return page
+
+
+@ActionFactory.register(name=BrowserAction.GO_TO_URL.value.name,
+                        desc=BrowserAction.GO_TO_URL.value.desc,
+                        tool_name="browser")
+class GotoUrl(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.GO_TO_URL.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.GO_TO_URL.name} page is none")
+            return ActionResult(content="no page", keep=True), page
+
+        params = action.params
+        url = params.get("url")
+        if not url:
+            logger.warning("empty url, go to nothing.")
+            return ActionResult(content="empty url", keep=True), page
+        items = url.split('://')
+        if len(items) == 1:
+            if items[0][0] != '/':
+                url = "file://" + os.path.join(os.getcwd(), url)
+
+        page.goto(url)
+        page.wait_for_load_state()
+        msg = f'Navigated to {url}'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.GO_TO_URL.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.GO_TO_URL.name} page is none")
+            return ActionResult(content="no page", keep=True), page
+
+        url = action.params.get("url")
+        if not url:
+            logger.warning("empty url, go to nothing.")
+            return ActionResult(content="empty url", keep=True), page
+
+        items = url.split('://')
+        if len(items) == 1:
+            if items[0][0] != '/':
+                url = "file://" + os.path.join(os.getcwd(), url)
+
+        await page.goto(url)
+        await page.wait_for_load_state()
+        msg = f'Navigated to {url}'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+
+@ActionFactory.register(name=BrowserAction.INPUT_TEXT.value.name,
+                        desc=BrowserAction.INPUT_TEXT.value.desc,
+                        tool_name="browser")
+class InputText(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.INPUT_TEXT.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.INPUT_TEXT.name} page is none")
+            return ActionResult(content="input text no page", keep=True), page
+
+        params = action.params
+        index = params.get("index", 0)
+        # compatible with int and str datatype
+        index = int(index)
+        input = params.get("text", "")
+
+        ob: Observation = kwargs.get("observation")
+        if not ob or index not in ob.dom_tree.element_map:
+            raise RuntimeError(f'Element index {index} does not exist')
+        if not input:
+            raise ValueError(f'No input to the page')
+
+        element_node = ob.dom_tree.element_map[index]
+        self.input_to_element(input, page, element_node)
+        msg = f'Input {input} into index {index}'
+        logger.info(f"action {msg}")
+        logger.debug(f'Element xpath: {element_node.xpath}')
+        return ActionResult(content=msg, keep=True), page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.INPUT_TEXT.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.INPUT_TEXT.name} page is none")
+            return ActionResult(content="input text no page", keep=True), page
+
+        params = action.params
+        index = params.get("index")
+        # compatible with int and str datatype
+        index = int(index)
+        input = params.get("text", "")
+
+        ob: Observation = kwargs.get("observation")
+        if not ob or index not in ob.dom_tree.element_map:
+            raise RuntimeError(f'Element index {index} does not exist')
+        if not input:
+            raise ValueError(f'No input to the page')
+
+        element_node = ob.dom_tree.element_map[index]
+        await self.async_input_to_element(input, page, element_node)
+        msg = f'Input {input} into index {index}'
+        logger.info(f"action {msg}")
+        logger.debug(f'Element xpath: {element_node.xpath}')
+        return ActionResult(content=msg, keep=True), page
+
+    def input_to_element(self, input: str, page, element_node: DOMElementNode):
+        try:
+            # Highlight before typing
+            # if element_node.highlight_index is not None:
+            # 	await self._update_state(focus_element=element_node.highlight_index)
+
+            element_handle = DomUtil.get_locate_element(page, element_node)
+
+            if element_handle is None:
+                raise RuntimeError(f'Element: {repr(element_node)} not found')
+
+            # Ensure element is ready for input
+            try:
+                element_handle.wait_for_element_state('stable', timeout=1000)
+                element_handle.scroll_into_view_if_needed(timeout=1000)
+            except Exception:
+                pass
+
+            # Get element properties to determine input method
+            is_contenteditable = element_handle.get_property('isContentEditable')
+
+            # Different handling for contenteditable vs input fields
+            if is_contenteditable.json_value():
+                element_handle.evaluate('el => el.textContent = ""')
+                element_handle.type(input, delay=5)
+            else:
+                element_handle.fill(input)
+
+        except Exception as e:
+            logger.warning(f'Failed to input text into element: {repr(element_node)}. Error: {str(e)}')
+            raise RuntimeError(f'Failed to input text into index {element_node.highlight_index}')
+
+    async def async_input_to_element(self, input: str, page, element_node: DOMElementNode):
+        try:
+            element_handle = await DomUtil.async_get_locate_element(page, element_node)
+
+            if element_handle is None:
+                raise RuntimeError(f'Element: {repr(element_node)} not found')
+
+            # Ensure element is ready for input
+            try:
+                await element_handle.wait_for_element_state('stable', timeout=1000)
+                await element_handle.scroll_into_view_if_needed(timeout=1000)
+            except Exception:
+                pass
+
+            # Get element properties to determine input method
+            is_contenteditable = await element_handle.get_property('isContentEditable')
+
+            # Different handling for contenteditable vs input fields
+            if await is_contenteditable.json_value():
+                await element_handle.evaluate('el => el.textContent = ""')
+                await element_handle.type(input, delay=5)
+            else:
+                await element_handle.fill(input)
+        except Exception as e:
+            logger.warning(f'Failed to input text into element: {repr(element_node)}. Error: {str(e)}')
+            raise RuntimeError(f'Failed to input text into index {element_node.highlight_index}')
+
+
+@ActionFactory.register(name=BrowserAction.CLICK_ELEMENT.value.name,
+                        desc=BrowserAction.CLICK_ELEMENT.value.desc,
+                        tool_name="browser")
+class ClickElement(ExecutableAction):
+    def __init__(self):
+        import_packages(['playwright', 'markdownify'])
+
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        from playwright.sync_api import BrowserContext
+
+        logger.info(f"exec {BrowserAction.CLICK_ELEMENT.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.CLICK_ELEMENT.name} page is none")
+            return ActionResult(content="input text no page", keep=True), page
+
+        browser: BrowserContext = get_browser(**kwargs)
+        if browser is None:
+            logger.warning(f"{BrowserAction.CLICK_ELEMENT.name} browser context is none")
+            return ActionResult(content="none browser context", keep=True), page
+
+        index = action.params.get("index")
+        # compatible with int and str datatype
+        index = int(index)
+        ob: Observation = kwargs.get("observation")
+        if not ob or index not in ob.dom_tree.element_map:
+            raise RuntimeError(f'Element index {index} does not exist')
+        if not input:
+            raise ValueError(f'No input to the page')
+        element_node = ob.dom_tree.element_map[index]
+
+        try:
+            pages = len(browser.pages)
+            msg = f'Clicked button with index {index}: {element_node.get_all_text_till_next_clickable_element(max_depth=2)}'
+            logger.info(msg)
+
+            DomUtil.click_element(page, element_node, browser=browser)
+            logger.debug(f'Element xpath: {element_node.xpath}')
+            if len(browser.pages) > pages:
+                new_tab_msg = 'Open the new tab'
+                msg += f' - {new_tab_msg}'
+                logger.info(new_tab_msg)
+                page = browser.pages[-1]
+                page.bring_to_front()
+                page.wait_for_load_state(timeout=60000)
+            return ActionResult(content=msg, keep=True), page
+        except Exception as e:
+            logger.warning(f'Element not clickable with index {index} - most likely the page changed')
+            return ActionResult(error=str(e)), page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.CLICK_ELEMENT.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warn(f"{BrowserAction.CLICK_ELEMENT.name} page is none")
+            return ActionResult(content="input text no page", keep=True), page
+
+        browser = get_browser(**kwargs)
+        if browser is None:
+            logger.warning(f"{BrowserAction.CLICK_ELEMENT.name} browser context is none")
+            return ActionResult(content="none browser context", keep=True), page
+
+        index = action.params.get("index")
+        # compatible with int and str datatype
+        index = int(index)
+        ob: Observation = kwargs.get("observation")
+        if not ob or index not in ob.dom_tree.element_map:
+            raise RuntimeError(f'Element index {index} does not exist')
+        if not input:
+            raise ValueError(f'No input to the page')
+        element_node = ob.dom_tree.element_map[index]
+        pages = len(browser.pages)
+
+        try:
+            await DomUtil.async_click_element(page, element_node, browser=browser)
+            msg = f'Clicked button with index {index}: {element_node.get_all_text_till_next_clickable_element(max_depth=2)}'
+
+            logger.info(msg)
+            logger.debug(f'Element xpath: {element_node.xpath}')
+            if len(browser.pages) > pages:
+                new_tab_msg = 'Open the new tab'
+                msg += f' - {new_tab_msg}'
+                logger.info(new_tab_msg)
+                page = browser.pages[-1]
+                await page.bring_to_front()
+                await page.wait_for_load_state(timeout=60000)
+            return ActionResult(content=msg, keep=True), page
+        except Exception as e:
+            logger.warning(f'Element not clickable with index {index} - most likely the page changed')
+            return ActionResult(error=str(e)), page
+
+
+# SEARCH_ENGINE = {"": "https://www.google.com/search?udm=14&q=",
+#                  "google": "https://www.google.com/search?udm=14&q="}
+
+SEARCH_ENGINE = {"": "https://www.bing.com/search?q=",
+                 "google": "https://www.bing.com/search?q="}
+
+
+@ActionFactory.register(name=BrowserAction.SEARCH.value.name,
+                        desc=BrowserAction.SEARCH.value.desc,
+                        tool_name="browser")
+class Search(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SEARCH.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.SEARCH.name} page is none")
+            return ActionResult(content="search no page", keep=True), page
+
+        params = action.params if action.params else {}
+        engine = params.get("engine", "")
+        url = SEARCH_ENGINE.get(engine)
+        query = params.get("query")
+        page.goto(f'{url}{query}')
+        page.wait_for_load_state()
+        msg = f'Searched for "{query}" in {url}'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SEARCH.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.SEARCH.name} page is none")
+            return ActionResult(content="search no page", keep=True), page
+
+        params = action.params if action.params else {}
+        engine = params.get("engine", "")
+        url = SEARCH_ENGINE.get(engine)
+        query = params.get("query")
+        await page.goto(f'{url}{query}')
+        await page.wait_for_load_state()
+        msg = f'Searched for "{query}" in {url}'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+
+@ActionFactory.register(name=BrowserAction.SEARCH_GOOGLE.value.name,
+                        desc=BrowserAction.SEARCH_GOOGLE.value.desc,
+                        tool_name="browser")
+class SearchGoogle(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SEARCH_GOOGLE.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.SEARCH_GOOGLE.name} page is none")
+            return ActionResult(content="search no page", keep=True), page
+
+        query = action.params.get("query")
+        page.goto(f'{SEARCH_ENGINE.get("")}{query}')
+        page.wait_for_load_state()
+        msg = f'Searched for "{query}" in Google'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SEARCH_GOOGLE.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.SEARCH_GOOGLE.name} page is none")
+            return ActionResult(content="search no page", keep=True), page
+
+        query = action.params.get("query")
+        await page.goto(f'{SEARCH_ENGINE.get("")}{query}')
+        await page.wait_for_load_state()
+        msg = f'Searched for "{query}" in Google'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+
+@ActionFactory.register(name=BrowserAction.NEW_TAB.value.name,
+                        desc=BrowserAction.NEW_TAB.value.desc,
+                        tool_name="browser")
+class NewTab(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.NEW_TAB.value.name} action")
+        browser = get_browser(**kwargs)
+        url = action.params.get("url")
+
+        new_page = browser.new_page()
+        new_page.wait_for_load_state()
+
+        if url:
+            new_page.goto(url)
+            DomUtil.wait_for_stable_network(new_page)
+
+        msg = f'Opened new tab with {url}'
+        logger.debug(msg)
+        return ActionResult(content=msg, keep=True), new_page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.NEW_TAB.value.name} action")
+        browser = get_browser(**kwargs)
+        url = action.params.get("url")
+        new_page = await browser.new_page()
+        await new_page.wait_for_load_state()
+
+        if url:
+            await new_page.goto(url)
+            DomUtil.wait_for_stable_network(new_page)
+        msg = f'Opened new tab with {url}'
+        logger.debug(msg)
+        return ActionResult(content=msg, keep=True), get_page(**kwargs)
+
+
+@ActionFactory.register(name=BrowserAction.GO_BACK.value.name,
+                        desc=BrowserAction.GO_BACK.value.desc,
+                        tool_name="browser")
+class GoBack(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.GO_BACK.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.GO_BACK.name} page is none")
+            return ActionResult(content="search no page", keep=True), page
+
+        page.go_back()
+        msg = 'Navigated back'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.GO_BACK.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.GO_BACK.name} page is none")
+            return ActionResult(content="search no page", keep=True), page
+
+        await page.go_back()
+        msg = 'Navigated back'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+
+@ActionFactory.register(name=BrowserAction.EXTRACT_CONTENT.value.name,
+                        desc=BrowserAction.EXTRACT_CONTENT.value.desc,
+                        tool_name="browser")
+class ExtractContent(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        import markdownify
+        from langchain_core.prompts import PromptTemplate
+
+        logger.info(f"exec {BrowserAction.EXTRACT_CONTENT.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.EXTRACT_CONTENT.name} page is none")
+            return ActionResult(content="extract content no page", keep=True), page
+
+        goal = action.params.get("goal")
+        llm_config = kwargs.get("llm_config")
+        if llm_config and llm_config.llm_api_key:
+            llm = get_llm_model(llm_config)
+        max_extract_content_output_tokens = kwargs.get("max_extract_content_output_tokens")
+        max_extract_content_input_tokens = kwargs.get("max_extract_content_input_tokens")
+
+        content = markdownify.markdownify(page.content())
+
+        # Truncate content if it exceeds max input tokens
+        if max_extract_content_input_tokens and len(content) > max_extract_content_input_tokens:
+            logger.warning(
+                f"Content length ({len(content)}) exceeds max input tokens ({max_extract_content_input_tokens}). Truncating content.")
+            content = content[:max_extract_content_input_tokens]
+
+        prompt = 'Your task is to extract the content of the page. You will be given a page and a goal and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. Extraction goal: {goal}, Page: {page}'
+        prompt_with_outputlimit = 'Your task is to extract the content of the page. You will be given a page and a goal and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. Extraction goal: {goal}, Page: {page} \n\n#The length of the returned result must be less than {max_extract_content_output_tokens} characters.'
+        template = PromptTemplate(input_variables=['goal', 'page'], template=prompt)
+
+        messages = [{'role': 'user', 'content': template.format(goal=goal, page=content)}]
+        try:
+            output = call_llm_model(llm,
+                                    messages=messages,
+                                    model=llm_config.llm_model_name,
+                                    temperature=llm_config.llm_temperature)
+            result_content = output.content
+
+            # Check if output exceeds the token limit and retry with length-limited prompt if needed
+            if max_extract_content_output_tokens and len(result_content) > max_extract_content_output_tokens:
+                logger.warning(
+                    f"Output exceeds maximum length ({len(result_content)} > {max_extract_content_output_tokens}). Retrying with limited prompt.")
+                template_with_limit = PromptTemplate(
+                    input_variables=['goal', 'page', 'max_extract_content_output_tokens'],
+                    template=prompt_with_outputlimit
+                )
+                messages = [{'role': 'user', 'content': template_with_limit.format(
+                    goal=goal,
+                    page=content,
+                    max_extract_content_output_tokens=max_extract_content_output_tokens,
+                    max_tokens=max_extract_content_output_tokens
+                )}]
+                # extract content with length limit
+                output = call_llm_model(llm,
+                                        messages=messages,
+                                        model=llm_config.llm_model_name,
+                                        temperature=llm_config.llm_temperature)
+                result_content = output.content
+
+            msg = f'Extracted from page\n: {result_content}\n'
+            logger.info(msg)
+            return ActionResult(content=msg, keep=True), page
+        except Exception as e:
+            logger.debug(f'Error extracting content: {e}')
+            msg = f'Extracted from page\n: {content}\n'
+            logger.info(msg)
+            return ActionResult(content=msg), page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        import markdownify
+        from langchain_core.prompts import PromptTemplate
+
+        logger.info(f"exec {BrowserAction.EXTRACT_CONTENT.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.EXTRACT_CONTENT.name} page is none")
+            return ActionResult(content="extract content no page", keep=True), page
+
+        goal = action.params.get("goal")
+        llm_config = kwargs.get("llm_config")
+        if llm_config and llm_config.llm_api_key:
+            llm = get_llm_model(llm_config)
+        content = markdownify.markdownify(await page.content())
+        max_extract_content_output_tokens = kwargs.get("max_extract_content_output_tokens")
+        max_extract_content_input_tokens = kwargs.get("max_extract_content_input_tokens")
+
+        # Truncate content if it exceeds max input tokens
+        if max_extract_content_input_tokens and len(content) > max_extract_content_input_tokens:
+            logger.warning(
+                f"Content length ({len(content)}) exceeds max input tokens ({max_extract_content_input_tokens}). Truncating content.")
+            content = content[:max_extract_content_input_tokens]
+
+        prompt = 'Your task is to extract the content of the page. You will be given a page and a goal and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. Extraction goal: {goal}, Page: {page}'
+        prompt_with_outputlimit = 'Your task is to extract the content of the page. You will be given a page and a goal and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. Extraction goal: {goal}, Page: {page} \n\n#The length of the returned result must be less than {max_extract_content_output_tokens} characters.'
+        template = PromptTemplate(input_variables=['goal', 'page'], template=prompt)
+
+        messages = [{'role': 'user', 'content': template.format(goal=goal, page=content)}]
+        try:
+            output = call_llm_model(llm,
+                                    messages=messages,
+                                    model=llm_config.llm_model_name,
+                                    temperature=llm_config.llm_temperature)
+            result_content = output.content
+
+            # Check if output exceeds the token limit and retry with length-limited prompt if needed
+            if max_extract_content_output_tokens and len(result_content) > max_extract_content_output_tokens:
+                logger.info(
+                    f"Output exceeds maximum length ({len(result_content)} > {max_extract_content_output_tokens}). Retrying with limited prompt.")
+                template_with_limit = PromptTemplate(
+                    input_variables=['goal', 'page', 'max_extract_content_output_tokens'],
+                    template=prompt_with_outputlimit
+                )
+                messages = [{'role': 'user', 'content': template_with_limit.format(
+                    goal=goal,
+                    page=content,
+                    max_extract_content_output_tokens=max_extract_content_output_tokens,
+                    max_tokens=max_extract_content_output_tokens
+                )}]
+                # extract content with length limit
+                output = call_llm_model(llm,
+                                        messages=messages,
+                                        model=llm_config.llm_model_name,
+                                        temperature=llm_config.llm_temperature)
+                result_content = output.content
+
+            msg = f'Extracted from page\n: {result_content}\n'
+            logger.info(msg)
+            return ActionResult(content=msg, keep=True), page
+        except Exception as e:
+            logger.debug(f'Error extracting content: {e}')
+            msg = f'Extracted from page\n: {content}\n'
+            logger.info(msg)
+            return ActionResult(content=msg), page
+
+
+@ActionFactory.register(name=BrowserAction.SCROLL_DOWN.value.name,
+                        desc=BrowserAction.SCROLL_DOWN.value.desc,
+                        tool_name="browser")
+class ScrollDown(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SCROLL_DOWN.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.SCROLL_DOWN.name} page is none")
+            return ActionResult(content="scroll no page", keep=True), page
+
+        amount = action.params.get("amount")
+        if not amount:
+            page.evaluate('window.scrollBy(0, window.innerHeight);')
+        else:
+            amount = int(amount)
+            page.evaluate(f'window.scrollBy(0, {amount});')
+
+        amount = f'{amount} pixels' if amount else 'one page'
+        msg = f'Scrolled down the page by {amount}'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SCROLL_DOWN.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.SCROLL_DOWN.name} page is none")
+            return ActionResult(content="scroll no page", keep=True), page
+
+        amount = action.params.get("amount")
+        if not amount:
+            await page.evaluate('window.scrollBy(0, window.innerHeight);')
+        else:
+            amount = int(amount)
+            await page.evaluate(f'window.scrollBy(0, {amount});')
+
+        amount = f'{amount} pixels' if amount else 'one page'
+        msg = f'Scrolled down the page by {amount}'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+
+@ActionFactory.register(name=BrowserAction.SCROLL_UP.value.name,
+                        desc=BrowserAction.SCROLL_UP.value.desc,
+                        tool_name="browser")
+class ScrollUp(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SCROLL_UP.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.SCROLL_UP.name} page is none")
+            return ActionResult(content="scroll no page", keep=True), page
+
+        amount = action.params.get("amount")
+        if not amount:
+            page.evaluate('window.scrollBy(0, -window.innerHeight);')
+        else:
+            amount = int(amount)
+            page.evaluate(f'window.scrollBy(0, -{amount});')
+
+        amount = f'{amount} pixels' if amount else 'one page'
+        msg = f'Scrolled down the page by {amount}'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SCROLL_UP.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.SCROLL_UP.name} page is none")
+            return ActionResult(content="scroll no page", keep=True), page
+
+        amount = action.params.get("amount")
+        if not amount:
+            await page.evaluate('window.scrollBy(0, -window.innerHeight);')
+        else:
+            amount = int(amount)
+            await page.evaluate(f'window.scrollBy(0, -{amount});')
+
+        amount = f'{amount} pixels' if amount else 'one page'
+        msg = f'Scrolled down the page by {amount}'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+
+@ActionFactory.register(name=BrowserAction.WAIT.value.name,
+                        desc=BrowserAction.WAIT.value.desc,
+                        tool_name="browser")
+class Wait(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        seconds = action.params.get("seconds")
+        if not seconds:
+            seconds = action.params.get("duration", 0)
+        seconds = int(seconds)
+        msg = f'Waiting for {seconds} seconds'
+        logger.info(msg)
+        time.sleep(seconds)
+        return ActionResult(content=msg, keep=True), kwargs.get('page')
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        seconds = action.params.get("seconds")
+        if not seconds:
+            seconds = action.params.get("duration", 0)
+        seconds = int(seconds)
+        msg = f'Waiting for {seconds} seconds'
+        logger.info(msg)
+        await asyncio.sleep(seconds)
+        return ActionResult(content=msg, keep=True), kwargs.get('page')
+
+
+@ActionFactory.register(name=BrowserAction.SWITCH_TAB.value.name,
+                        desc=BrowserAction.SWITCH_TAB.value.desc,
+                        tool_name="browser")
+class SwitchTab(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SWITCH_TAB.value.name} action")
+        browser = get_browser(**kwargs)
+        if browser is None:
+            logger.warning(f"{BrowserAction.SWITCH_TAB.name} browser context is none")
+            return ActionResult(content="switch tab no browser context", keep=True), get_page(**kwargs)
+
+        page_id = action.params.get("page_id", 0)
+        page_id = int(page_id)
+        pages = browser.pages
+
+        if page_id >= len(pages):
+            raise RuntimeError(f'No tab found with page_id: {page_id}')
+
+        page = pages[page_id]
+        page.bring_to_front()
+        page.wait_for_load_state()
+        msg = f'Switched to tab {page_id}'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SWITCH_TAB.value.name} action")
+        browser = get_browser(**kwargs)
+        if browser is None:
+            logger.warning(f"{BrowserAction.SWITCH_TAB.name} browser context is none")
+            return ActionResult(content="switch tab no browser context", keep=True), get_page(**kwargs)
+
+        page_id = action.params.get("page_id", 0)
+        page_id = int(page_id)
+        pages = browser.pages
+
+        if page_id >= len(pages):
+            raise RuntimeError(f'No tab found with page_id: {page_id}')
+
+        page = pages[page_id]
+        await page.bring_to_front()
+        await page.wait_for_load_state()
+        msg = f'Switched to tab {page_id}'
+        logger.info(msg)
+        return ActionResult(content=msg, keep=True), page
+
+
+@ActionFactory.register(name=BrowserAction.SEND_KEYS.value.name,
+                        desc=BrowserAction.SEND_KEYS.value.desc,
+                        tool_name="browser")
+class SendKeys(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SEND_KEYS.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.SEND_KEYS.name} page is none")
+            return ActionResult(content="scroll no page", keep=True), page
+
+        keys = action.params.get("keys")
+        if not keys:
+            return ActionResult(success=False, content="no keys", keep=True), page
+
+        try:
+            page.keyboard.press(keys)
+        except Exception as e:
+            logger.warning(f"{keys} press fail. \n{traceback.format_exc()}")
+            raise e
+        return ActionResult(content=f"Sent keys: {keys}", keep=True), page
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.SEND_KEYS.value.name} action")
+        page = get_page(**kwargs)
+        if page is None:
+            logger.warning(f"{BrowserAction.SEND_KEYS.name} page is none")
+            return ActionResult(content="scroll no page", keep=True), page
+
+        keys = action.params.get("keys")
+        if not keys:
+            return ActionResult(success=False, content="no keys", keep=True), page
+
+        try:
+            await page.keyboard.press(keys)
+        except Exception as e:
+            logger.warning(f"{keys} press fail. \n{traceback.format_exc()}")
+            raise e
+
+        return ActionResult(content=f"Sent keys: {keys}", keep=True), page
+
+
+@ActionFactory.register(name=BrowserAction.WRITE_TO_FILE.value.name,
+                        desc=BrowserAction.WRITE_TO_FILE.value.desc,
+                        tool_name="browser")
+class WriteToFile(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        # 设置默认文件路径
+        file_path = "tmp_result.md"
+        # 检查参数中是否有file_path
+        if "file_path" in action.params:
+            file_path = action.params.get("file_path", "tmp_result.md")
+        # 检查参数中是否有file_name
+        elif "file_name" in action.params:
+            file_path = action.params.get("file_name", "tmp_result.md")
+        elif "filename" in action.params:
+            file_path = action.params.get("filename", "tmp_result.md")
+        content = action.params.get("content", "")
+        mode = action.params.get("mode", "a")  # Default to append mode
+        # 获取文件的绝对路径
+        abs_file_path = os.path.abspath(file_path)
+        try:
+            with open(file_path, mode, encoding='utf-8') as f:
+                f.write(content + '\n')
+            msg = f'Successfully wrote content to {abs_file_path}'
+            logger.info(msg)
+            return ActionResult(content=msg, keep=True), get_page(**kwargs)
+        except Exception as e:
+            error_msg = f'Failed to write to file {abs_file_path}: {str(e)}'
+            logger.error(error_msg)
+            return ActionResult(content=error_msg, keep=True, error=error_msg), get_page(**kwargs)
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        # For file operations, we don't need to make this asynchronous
+        return self.act(action, **kwargs)
+
+
+@ActionFactory.register(name=BrowserAction.DONE.value.name,
+                        desc=BrowserAction.DONE.value.desc,
+                        tool_name="browser")
+class Done(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.DONE.value.name} action")
+        return ActionResult(is_done=True, success=True, content="done", keep=True), get_page(**kwargs)
+
+    async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info(f"exec {BrowserAction.DONE.value.name} action")
+        return ActionResult(is_done=True, success=True, content="done", keep=True), get_page(**kwargs)

chapter9/gaia-experience/AWorld/examples/common/tools/browsers/action/executor.py

@@ -0,0 +1,71 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+from typing import Tuple, List, Any
+
+from aworld.core.tool.action_factory import ActionFactory
+from aworld.core.common import ActionModel, ActionResult, Observation
+from aworld.logs.util import logger
+from aworld.core.tool.base import Tool, ToolActionExecutor
+
+
+class BrowserToolActionExecutor(ToolActionExecutor):
+    def __init__(self, tool: Tool = None):
+        super(BrowserToolActionExecutor, self).__init__(tool)
+
+    def execute_action(self, actions: List[ActionModel], **kwargs) -> Tuple[
+        List[ActionResult], Any]:
+        """Execute the specified browser action sequence by agent policy.
+
+        Args:
+            actions: Tool action sequence.
+
+        Returns:
+            Browser page and action result list.
+        """
+        action_results = []
+        page = self.tool.page
+        for action in actions:
+            action_result, page = self._exec(action, **kwargs)
+            action_results.append(action_result)
+        return action_results, page
+
+    async def async_execute_action(self, actions: List[ActionModel], **kwargs) -> Tuple[
+        List[ActionResult], Any]:
+        """Execute the specified browser action sequence by agent policy.
+
+        Args:
+            actions: Tool action sequence.
+
+        Returns:
+            Browser page and action result list.
+        """
+        action_results = []
+        page = self.tool.page
+        for action in actions:
+            action_result, page = await self._async_exec(action, **kwargs)
+            action_results.append(action_result)
+        return action_results, page
+
+    def _exec(self, action_model: ActionModel, **kwargs):
+        action_name = action_model.action_name
+        if action_name not in ActionFactory:
+            raise ValueError(f'Action {action_name} not found')
+
+        action = ActionFactory(action_name)
+        action_result, page = action.act(action_model, page=self.tool.page, browser=self.tool.browser_context, **kwargs)
+        logger.info(f"{action_name} execute finished")
+        return action_result, page
+
+    async def _async_exec(self, action_model: ActionModel, **kwargs):
+        action_name = action_model.action_name
+        if action_name not in ActionFactory:
+            action_name = action_model.tool_name + action_model.action_name
+            if action_name not in ActionFactory:
+                raise ValueError(f'Action {action_name} not found')
+
+        action = ActionFactory(action_name)
+        action_result, page = await action.async_act(action_model, page=self.tool.page,
+                                                     browser=self.tool.browser_context, **kwargs)
+        logger.info(f"{action_name} execute finished")
+        return action_result, page

chapter9/gaia-experience/AWorld/examples/common/tools/browsers/action/utils.py

@@ -0,0 +1,507 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+import re
+import time
+import traceback
+from typing import Optional
+
+from examples.common.tools.browsers.util.dom import DOMElementNode
+from aworld.logs.util import logger
+from aworld.utils import import_package
+
+
+class DomUtil:
+    def __init__(self):
+        import_package("playwright")
+
+    @staticmethod
+    async def async_click_element(page, element_node: DOMElementNode, **kwargs) -> Optional[str]:
+        from playwright.async_api import ElementHandle as AElementHandle, BrowserContext as ABrowserContext
+
+        try:
+            element_handle: AElementHandle = await DomUtil.async_get_locate_element(page, element_node)
+            if element_handle is None:
+                raise Exception(f'Element: {repr(element_node)} not found')
+
+            bound = await element_handle.bounding_box()
+            try:
+                # todo: iframe.
+                center_x = bound['x'] + bound['width'] / 2
+                center_y = bound['y'] + bound['height'] / 2
+
+                try:
+                    browser: ABrowserContext = kwargs.get('browser')
+                    async with browser.expect_page() as new_page_info:
+                        await page.mouse.click(center_x, center_y)
+                    await page.mouse.click(center_x, center_y)
+                    await page.wait_for_load_state()
+                except:
+                    logger.warning(traceback.format_exc())
+            except:
+                logger.info(f"click {element_handle}!!")
+                if await element_handle.text_content():
+                    browser: ABrowserContext = kwargs.get('browser')
+                    if browser:
+                        try:
+                            async with browser.expect_page() as new_page_info:
+                                await page.click(f"text={element_handle.text_content()}")
+                            page = await new_page_info.value
+                            await page.wait_for_load_state()
+                        except:
+                            logger.warning(traceback.format_exc())
+                    else:
+                        await element_handle.click()
+                        await page.wait_for_load_state()
+                else:
+                    await element_handle.click()
+                    await page.wait_for_load_state()
+        except Exception as e:
+            logger.error(traceback.format_exc())
+            raise Exception(f'Failed to click element: {repr(element_node)}. Error: {str(e)}')
+
+    @staticmethod
+    def click_element(page, element_node: DOMElementNode, **kwargs) -> Optional[str]:
+        from playwright.sync_api import ElementHandle, BrowserContext
+
+        try:
+            element_handle: ElementHandle = DomUtil.get_locate_element(page, element_node)
+            if element_handle is None:
+                raise Exception(f'Element: {repr(element_node)} not found')
+
+            bound = element_handle.bounding_box()
+            try:
+                # todo: iframe.
+                center_x = bound['x'] + bound['width'] / 2
+                center_y = bound['y'] + bound['height'] / 2
+
+                try:
+                    browser: BrowserContext = kwargs.get('browser')
+                    with browser.expect_page() as new_page_info:
+                        page.mouse.click(center_x, center_y)
+                    page = new_page_info.value
+                    page.wait_for_load_state()
+                except:
+                    logger.warning(traceback.format_exc())
+            except:
+                logger.info(f"click {element_handle}!!")
+                if element_handle.text_content():
+                    browser: BrowserContext = kwargs.get('browser')
+                    if browser:
+                        try:
+                            with browser.expect_page() as new_page_info:
+                                page.click(f"text={element_handle.text_content()}")
+                            page = new_page_info.value
+                            page.wait_for_load_state()
+                        except:
+                            logger.warning(traceback.format_exc())
+                    else:
+                        element_handle.click()
+                        page.wait_for_load_state()
+                else:
+                    element_handle.click()
+                    page.wait_for_load_state()
+        except Exception as e:
+            logger.error(traceback.format_exc())
+            raise Exception(f'Failed to click element: {repr(element_node)}. Error: {str(e)}')
+
+    @staticmethod
+    async def async_get_locate_element(current_frame, element: DOMElementNode):
+        # Start with the target element and collect all parents, return Optional[AElementHandle]
+        from playwright.async_api import FrameLocator as AFrameLocator
+
+        parents: list[DOMElementNode] = []
+        current = element
+        while current.parent is not None:
+            parent = current.parent
+            parents.append(parent)
+            current = parent
+
+        # Reverse the parents list to process from top to bottom
+        parents.reverse()
+
+        # Process all iframe parents in sequence
+        iframes = [item for item in parents if item.tag_name == 'iframe']
+        for parent in iframes:
+            css_selector = DomUtil._enhanced_css_selector_for_element(
+                parent,
+                include_dynamic_attributes=True,
+            )
+            current_frame = current_frame.frame_locator(css_selector)
+
+        css_selector = DomUtil._enhanced_css_selector_for_element(
+            element, include_dynamic_attributes=True
+        )
+
+        try:
+            if isinstance(current_frame, AFrameLocator):
+                element_handle = await current_frame.locator(css_selector).element_handle()
+                return element_handle
+            else:
+                # Try to scroll into view if hidden
+                element_handle = await current_frame.query_selector(css_selector)
+                if element_handle:
+                    await element_handle.scroll_into_view_if_needed()
+                    return element_handle
+                return None
+        except Exception as e:
+            logger.error(f'Failed to locate element: {str(e)}')
+            return None
+
+    @staticmethod
+    def get_locate_element(current_frame, element: DOMElementNode):
+        # Start with the target element and collect all parents
+        from playwright.sync_api import FrameLocator
+
+        parents: list[DOMElementNode] = []
+        current = element
+        while current.parent is not None:
+            parent = current.parent
+            parents.append(parent)
+            current = parent
+
+        # Reverse the parents list to process from top to bottom
+        parents.reverse()
+
+        # Process all iframe parents in sequence
+        iframes = [item for item in parents if item.tag_name == 'iframe']
+        for parent in iframes:
+            css_selector = DomUtil._enhanced_css_selector_for_element(
+                parent,
+                include_dynamic_attributes=True,
+            )
+            current_frame = current_frame.frame_locator(css_selector)
+
+        css_selector = DomUtil._enhanced_css_selector_for_element(
+            element, include_dynamic_attributes=True
+        )
+
+        try:
+            if isinstance(current_frame, FrameLocator):
+                element_handle = current_frame.locator(css_selector).element_handle()
+                return element_handle
+            else:
+                # Try to scroll into view if hidden
+                element_handle = current_frame.query_selector(css_selector)
+                if element_handle:
+                    element_handle.scroll_into_view_if_needed()
+                    return element_handle
+                return None
+        except Exception as e:
+            logger.error(f'Failed to locate element: {str(e)}')
+            return None
+
+    @staticmethod
+    def wait_for_stable_network(page, **kwargs):
+        pending_requests = set()
+        last_activity = time.time()
+
+        # Define relevant resource types and content types
+        RELEVANT_RESOURCE_TYPES = {
+            'document',
+            'stylesheet',
+            'image',
+            'font',
+            'script',
+            'iframe',
+        }
+
+        RELEVANT_CONTENT_TYPES = {
+            'text/html',
+            'text/css',
+            'application/javascript',
+            'image/',
+            'font/',
+            'application/json',
+        }
+
+        # Additional patterns to filter out
+        IGNORED_URL_PATTERNS = {
+            # Analytics and tracking
+            'analytics',
+            'tracking',
+            'telemetry',
+            'beacon',
+            'metrics',
+            # Ad-related
+            'doubleclick',
+            'adsystem',
+            'adserver',
+            'advertising',
+            # Social media widgets
+            'facebook.com/plugins',
+            'platform.twitter',
+            'linkedin.com/embed',
+            # Live chat and support
+            'livechat',
+            'zendesk',
+            'intercom',
+            'crisp.chat',
+            'hotjar',
+            # Push notifications
+            'push-notifications',
+            'onesignal',
+            'pushwoosh',
+            # Background sync/heartbeat
+            'heartbeat',
+            'ping',
+            'alive',
+            # WebRTC and streaming
+            'webrtc',
+            'rtmp://',
+            'wss://',
+            # Common CDNs for dynamic content
+            'cloudfront.net',
+            'fastly.net',
+        }
+
+        def on_request(request):
+            # Filter by resource type
+            if request.resource_type not in RELEVANT_RESOURCE_TYPES:
+                return
+
+            # Filter out streaming, websocket, and other real-time requests
+            if request.resource_type in {
+                'websocket',
+                'media',
+                'eventsource',
+                'manifest',
+                'other',
+            }:
+                return
+
+            # Filter out by URL patterns
+            url = request.url.lower()
+            if any(pattern in url for pattern in IGNORED_URL_PATTERNS):
+                return
+
+            # Filter out data URLs and blob URLs
+            if url.startswith(('data:', 'blob:')):
+                return
+
+            # Filter out requests with certain headers
+            headers = request.headers
+            if headers.get('purpose') == 'prefetch' or headers.get('sec-fetch-dest') in [
+                'video',
+                'audio',
+            ]:
+                return
+
+            nonlocal last_activity
+            pending_requests.add(request)
+            last_activity = time.time()
+
+        def on_response(response):
+            request = response.request
+            if request not in pending_requests:
+                return
+
+            # Filter by content type if available
+            content_type = response.headers.get('content-type', '').lower()
+
+            # Skip if content type indicates streaming or real-time data
+            if any(t in content_type
+                   for t in [
+                       'streaming',
+                       'video',
+                       'audio',
+                       'webm',
+                       'mp4',
+                       'event-stream',
+                       'websocket',
+                       'protobuf']):
+                pending_requests.remove(request)
+                return
+
+            # Only process relevant content types
+            if not any(ct in content_type for ct in RELEVANT_CONTENT_TYPES):
+                pending_requests.remove(request)
+                return
+
+            # Skip if response is too large (likely not essential for page load)
+            content_length = response.headers.get('content-length')
+            if content_length and int(content_length) > 5 * 1024 * 1024:  # 5MB
+                pending_requests.remove(request)
+                return
+
+            nonlocal last_activity
+            pending_requests.remove(request)
+            last_activity = time.time()
+
+        # Attach event listeners
+        page.on('request', on_request)
+        page.on('response', on_response)
+
+        try:
+            start_time = time.time()
+            while True:
+                time.sleep(0.1)
+                now = time.time()
+
+                if len(pending_requests) == 0 and (now - last_activity) >= kwargs.get('idle_wait_time', 0.5):
+                    break
+                if now - start_time > kwargs.get('max_wait_time', 5):
+                    logger.debug(
+                        f'Network timeout after {kwargs.get("max_wait_time", 5)}s with {len(pending_requests)} '
+                        f'pending requests: {[r.url for r in pending_requests]}'
+                    )
+                    break
+
+        finally:
+            # Clean up event listeners
+            page.remove_listener('request', on_request)
+            page.remove_listener('response', on_response)
+        logger.debug(f'Network stabilized for {kwargs.get("idle_wait_time", 0.5)} seconds')
+
+    @staticmethod
+    def _enhanced_css_selector_for_element(element: DOMElementNode, include_dynamic_attributes: bool = True) -> str:
+        """Creates a CSS selector for a DOM element, handling various edge cases and special characters.
+
+        Args:
+                element: The DOM element to create a selector for
+
+        Returns:
+                A valid CSS selector string
+        """
+        try:
+            # Get base selector from XPath
+            css_selector = DomUtil._convert_simple_xpath_to_css_selector(element.xpath)
+
+            # Handle class attributes
+            if 'class' in element.attributes and element.attributes['class'] and include_dynamic_attributes:
+                # Define a regex pattern for valid class names in CSS
+                valid_class_name_pattern = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_-]*$')
+
+                # Iterate through the class attribute values
+                classes = element.attributes['class'].split()
+                for class_name in classes:
+                    # Skip empty class names
+                    if not class_name.strip():
+                        continue
+
+                    # Check if the class name is valid
+                    if valid_class_name_pattern.match(class_name):
+                        # Append the valid class name to the CSS selector
+                        css_selector += f'.{class_name}'
+                    else:
+                        # Skip invalid class names
+                        continue
+
+            # Expanded set of safe attributes that are stable and useful for selection
+            SAFE_ATTRIBUTES = {
+                # Data attributes (if they're stable in your application)
+                'id',
+                # Standard HTML attributes
+                'name',
+                'type',
+                'placeholder',
+                # Accessibility attributes
+                'aria-label',
+                'aria-labelledby',
+                'aria-describedby',
+                'role',
+                # Common form attributes
+                'for',
+                'autocomplete',
+                'required',
+                'readonly',
+                # Media attributes
+                'alt',
+                'title',
+                'src',
+                # Custom stable attributes (add any application-specific ones)
+                'href',
+                'target',
+            }
+
+            if include_dynamic_attributes:
+                dynamic_attributes = {
+                    'data-id',
+                    'data-qa',
+                    'data-cy',
+                    'data-testid',
+                }
+                SAFE_ATTRIBUTES.update(dynamic_attributes)
+
+            # Handle other attributes
+            for attribute, value in element.attributes.items():
+                if attribute == 'class':
+                    continue
+
+                # Skip invalid attribute names
+                if not attribute.strip():
+                    continue
+
+                if attribute not in SAFE_ATTRIBUTES:
+                    continue
+
+                # Escape special characters in attribute names
+                safe_attribute = attribute.replace(':', r'\:')
+
+                # Handle different value cases
+                if value == '':
+                    css_selector += f'[{safe_attribute}]'
+                elif any(char in value for char in '"\'<>`\n\r\t'):
+                    # Use contains for values with special characters
+                    # Regex-substitute *any* whitespace with a single space, then strip.
+                    collapsed_value = re.sub(r'\s+', ' ', value).strip()
+                    # Escape embedded double-quotes.
+                    safe_value = collapsed_value.replace('"', '\\"')
+                    css_selector += f'[{safe_attribute}*="{safe_value}"]'
+                else:
+                    css_selector += f'[{safe_attribute}="{value}"]'
+
+            return css_selector
+
+        except Exception:
+            # Fallback to a more basic selector if something goes wrong
+            tag_name = element.tag_name or '*'
+            return f"{tag_name}[highlight_index='{element.highlight_index}']"
+
+    @staticmethod
+    def _convert_simple_xpath_to_css_selector(xpath: str) -> str:
+        """Converts simple XPath expressions to CSS selectors."""
+        if not xpath:
+            return ''
+
+        # Remove leading slash if present
+        xpath = xpath.lstrip('/')
+
+        # Split into parts
+        parts = xpath.split('/')
+        css_parts = []
+
+        for part in parts:
+            if not part:
+                continue
+
+            # Handle index notation [n]
+            if '[' in part:
+                base_part = part[: part.find('[')]
+                index_part = part[part.find('['):]
+
+                # Handle multiple indices
+                indices = [i.strip('[]') for i in index_part.split(']')[:-1]]
+
+                for idx in indices:
+                    try:
+                        # Handle numeric indices
+                        if idx.isdigit():
+                            index = int(idx) - 1
+                            base_part += f':nth-of-type({index + 1})'
+                        # Handle last() function
+                        elif idx == 'last()':
+                            base_part += ':last-of-type'
+                        # Handle position() functions
+                        elif 'position()' in idx:
+                            if '>1' in idx:
+                                base_part += ':nth-of-type(n+2)'
+                    except ValueError:
+                        continue
+
+                css_parts.append(base_part)
+            else:
+                css_parts.append(part)
+
+        base_selector = ' > '.join(css_parts)
+        return base_selector

chapter9/gaia-experience/AWorld/examples/common/tools/browsers/async_browser.py

@@ -0,0 +1,361 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+import asyncio
+import base64
+import json
+import os
+import subprocess
+import traceback
+from importlib import resources
+from pathlib import Path
+from typing import Any, Dict, Tuple, List
+
+from examples.common.tools.common import package
+from examples.common.tools.tool_action import BrowserAction
+from aworld.core.common import Observation, ActionModel, ActionResult
+from aworld.logs.util import logger
+from aworld.core.tool.base import action_executor, ToolFactory, AsyncTool
+from aworld.utils.import_package import is_package_installed
+from examples.common.tools.browsers.action.executor import BrowserToolActionExecutor
+from examples.common.tools.browsers.util.dom import DomTree
+from examples.common.tools.conf import BrowserToolConfig
+from examples.common.tools.browsers.util.dom_build import async_build_dom_tree
+from aworld.utils import import_package
+from aworld.tools.utils import build_observation
+
+URL_MAX_LENGTH = 4096
+UTF8 = "".join(chr(x) for x in range(0, 55290))
+ASCII = "".join(chr(x) for x in range(32, 128))
+
+
+@ToolFactory.register(name="browser",
+                      desc="browser",
+                      asyn=True,
+                      supported_action=BrowserAction,
+                      conf_file_name=f'browser_tool.yaml',
+                      dir=f"{Path(__file__).parent.absolute()}")
+class BrowserTool(AsyncTool):
+    def __init__(self, conf: BrowserToolConfig, **kwargs) -> None:
+        super(BrowserTool, self).__init__(conf)
+
+        self.initialized = False
+        self._finish = False
+        self.record_trace = self.conf.get("working_dir", False)
+        self.sleep_after_init = self.conf.get("sleep_after_init", False)
+        dom_js_path = self.conf.get('dom_js_path')
+        if dom_js_path and os.path.exists(dom_js_path):
+            with open(dom_js_path, 'r') as read:
+                self.js_code = read.read()
+        else:
+            self.js_code = resources.read_text(f'{package}.browsers.script',
+                                               'buildDomTree.js')
+        self.cur_observation = None
+        if not is_package_installed('playwright'):
+            import_package("playwright")
+            logger.info("playwright install...")
+            try:
+                subprocess.check_call('playwright install', shell=True, timeout=300)
+            except Exception as e:
+                logger.error(f"Fail to auto execute playwright install, you can install manually\n {e}")
+
+    async def init(self) -> None:
+        from playwright.async_api import async_playwright
+
+        if self.initialized:
+            return
+
+        self.context_manager = async_playwright()
+        self.playwright = await self.context_manager.start()
+
+        self.browser = await self._create_browser()
+        self.browser_context = await self._create_browser_context()
+
+        if self.record_trace:
+            await self.browser_context.tracing.start(screenshots=True, snapshots=True)
+
+        self.page = await self.browser_context.new_page()
+        if self.conf.get("custom_executor"):
+            self.action_executor = BrowserToolActionExecutor(self)
+        else:
+            self.action_executor = action_executor
+        self.initialized = True
+
+    async def _create_browser(self):
+        browse_name = self.conf.get("browse_name", "chromium")
+        browse = getattr(self.playwright, browse_name)
+        cdp_url = self.conf.get("cdp_url")
+        wss_url = self.conf.get("wss_url")
+        if cdp_url:
+            if browse_name != "chromium":
+                logger.warning(f"{browse_name} unsupported CDP, will use chromium browser")
+                browse = self.playwright.chromium
+            logger.info(f"Connecting to remote browser via CDP {cdp_url}")
+            browser = await browse.connect_over_cdp(cdp_url)
+        elif wss_url:
+            logger.info(f"Connecting to remote browser via wss {wss_url}")
+            browser = await browse.connect(wss_url)
+        else:
+            headless = self.conf.get("headless", False)
+            slow_mo = self.conf.get("slow_mo", 0)
+            disable_security_args = []
+            if self.conf.get('disable_security', False):
+                disable_security_args = ['--disable-web-security',
+                                         '--disable-site-isolation-trials',
+                                         '--disable-features=IsolateOrigins,site-per-process']
+            args = ['--no-sandbox',
+                    '--disable-crash-reporte',
+                    '--disable-blink-features=AutomationControlled',
+                    '--disable-infobars',
+                    '--disable-background-timer-throttling',
+                    '--disable-popup-blocking',
+                    '--disable-backgrounding-occluded-windows',
+                    '--disable-renderer-backgrounding',
+                    '--disable-window-activation',
+                    '--disable-focus-on-load',
+                    '--no-first-run',
+                    '--no-default-browser-check',
+                    '--no-startup-window',
+                    '--window-position=0,0',
+                    '--window-size=1280,720'] + disable_security_args
+            browser = await browse.launch(
+                headless=headless,
+                slow_mo=slow_mo,
+                args=args,
+                proxy=self.conf.get('proxy'),
+            )
+        return browser
+
+    async def _create_browser_context(self):
+        """Creates a new browser context with anti-detection measures and loads cookies if available."""
+        from playwright.async_api import ViewportSize
+
+        browser = self.browser
+        if self.conf.get("cdp_url") and len(browser.contexts) > 0:
+            context = browser.contexts[0]
+        else:
+            viewport_size = ViewportSize(width=self.conf.get("width", 1280),
+                                         height=self.conf.get("height", 720))
+            disable_security = self.conf.get('disable_security', False)
+
+            context = await browser.new_context(viewport=viewport_size,
+                                                no_viewport=False,
+                                                user_agent=self.conf.get('user_agent'),
+                                                java_script_enabled=True,
+                                                bypass_csp=disable_security,
+                                                ignore_https_errors=disable_security,
+                                                record_video_dir=self.conf.get('working_dir'),
+                                                record_video_size=viewport_size,
+                                                locale=self.conf.get('locale'),
+                                                storage_state=self.conf.get("storage_state", None),
+                                                geolocation=self.conf.get("geolocation", None),
+                                                device_scale_factor=1)
+            if "chromium" == self.conf.get("browse_name", "chromium"):
+                await context.grant_permissions(['camera', 'microphone'])
+
+        if self.conf.get('trace_path'):
+            await context.tracing.start(screenshots=True, snapshots=True, sources=True)
+
+        cookie_file = self.conf.get('cookies_file')
+        if cookie_file and os.path.exists(cookie_file):
+            with open(cookie_file, 'r') as read:
+                cookies = json.loads(read.read())
+                await context.add_cookies(cookies)
+                logger.info(f'Cookies load from {cookie_file} finished')
+
+        if self.conf.get('private'):
+            js = resources.read_text(f"{package}.browsers.script", "stealth.min.js")
+            await context.add_init_script(js)
+
+        return context
+
+    async def get_cur_page(self):
+        return self.page
+
+    async def screenshot(self, full_page: bool = False) -> str:
+        """Returns a base64 encoded screenshot of the current page.
+
+        Args:
+            full_page: When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport.
+
+        Returns:
+            Base64 of the page screenshot
+        """
+        page = await self.get_cur_page()
+
+        try:
+            await page.bring_to_front()
+            await page.wait_for_load_state(timeout=2000)
+        except:
+            logger.warning("bring to front load timeout")
+            pass
+
+        screenshot = await page.screenshot(
+            full_page=full_page,
+            animations='disabled',
+            timeout=600000
+        )
+        logger.info("page screenshot finished")
+        screenshot_base64 = base64.b64encode(screenshot).decode('utf-8')
+        return screenshot_base64
+
+    async def _get_observation(self, info: Dict[str, Any] = {}) -> Observation:
+        fail_error = info.get('exception')
+        if fail_error:
+            return Observation(observer=self.name(), action_result=[ActionResult(error=fail_error)])
+
+        try:
+            dom_tree = await self._parse_dom_tree()
+            image = await self.screenshot()
+            pixels_above, pixels_below = await self._scroll_info()
+            info.update({"pixels_above": pixels_above,
+                         "pixels_below": pixels_below,
+                         "url": self.page.url})
+            return Observation(observer=self.name(), dom_tree=dom_tree, image=image, info=info)
+        except Exception as e:
+            try:
+                try:
+                    await self.page.go_back()
+                except:
+                    logger.warning("current page abnormal, new page to use.")
+                    self.page = await self.browser_context.new_page()
+                dom_tree = await self._parse_dom_tree()
+                image = await self.screenshot()
+                pixels_above, pixels_below = await self._scroll_info()
+                info.update({"pixels_above": pixels_above,
+                             "pixels_below": pixels_below,
+                             "url": self.page.url})
+                return Observation(observer=self.name(), dom_tree=dom_tree, image=image, info=info)
+            except Exception as e:
+                logger.warning(f"build observation fail, {traceback.format_exc()}")
+                return Observation(observer=self.name(), action_result=[ActionResult(error=traceback.format_exc())])
+
+    async def _parse_dom_tree(self) -> DomTree:
+        args = {
+            'doHighlightElements': self.conf.get("do_highlight", True),
+            'focusHighlightIndex': self.conf.get("focus_highlight", -1),
+            'viewportExpansion': self.conf.get("viewport_expansion", 0),
+            'debugMode': logger.getEffectiveLevel() == 10,
+        }
+        element_tree, element_map = await async_build_dom_tree(self.page, self.js_code, args)
+        return DomTree(element_tree=element_tree, element_map=element_map)
+
+    async def _scroll_info(self) -> tuple[int, int]:
+        """Get scroll position information for the current page."""
+        scroll_y = await self.page.evaluate('window.scrollY')
+        viewport_height = await self.page.evaluate('window.innerHeight')
+        total_height = await self.page.evaluate('document.documentElement.scrollHeight')
+        pixels_above = scroll_y
+        pixels_below = total_height - (scroll_y + viewport_height)
+        return pixels_above, pixels_below
+
+    async def reset(self, *, seed: int | None = None, options: Dict[str, str] | None = None) -> Tuple[
+        Observation, Dict[str, Any]]:
+        await super().reset(seed=seed, options=options)
+        if self.initialized:
+            observation = await self._get_observation()
+            observation.action_result = [ActionResult(content='start', keep=True)]
+            self.cur_observation = observation
+            return observation, {}
+
+        await self.close()
+        await self.init()
+
+        if self.sleep_after_init > 0:
+            await asyncio.sleep(self.sleep_after_init)
+
+        observation = await self._get_observation()
+        observation.action_result = [ActionResult(content='start', keep=True)]
+        observation.ability = ''
+        self.cur_observation = observation
+        return observation, {}
+
+    async def save_trace(self, trace_path: str | Path) -> None:
+        if self.record_trace:
+            await self.browser_context.tracing.stop(path=trace_path)
+
+    @property
+    async def finished(self) -> bool:
+        return self._finish
+
+    async def close(self) -> None:
+        if hasattr(self, 'context') and self.browser_context:
+            await self.browser_context.close()
+        if hasattr(self, 'browser') and self.browser:
+            await self.browser.close()
+        if hasattr(self, 'playwright') and self.playwright:
+            await self.playwright.stop()
+        if self.initialized:
+            await self.context_manager.__aexit__()
+
+    async def do_step(self, action: List[ActionModel], **kwargs) -> Tuple[
+        Observation, float, bool, bool, Dict[str, Any]]:
+        if not self.initialized:
+            raise RuntimeError("Call init first before calling step.")
+
+        if not action:
+            logger.warning(f"{self.name()} has no action")
+            return build_observation(observer=self.name(), ability='', content='no action'), 0., False, False, {}
+
+        reward = 0
+        fail_error = ""
+        action_result = None
+
+        invalid_acts: List[int] = []
+        for i, act in enumerate(action):
+            if act.tool_name != 'browser':
+                logger.warning(f"tool {act.tool_name} is not a browser!")
+                invalid_acts.append(i)
+
+        if invalid_acts:
+            for i in invalid_acts:
+                action[i] = None
+
+        try:
+            action_result, self.page = await self.action_executor.async_execute_action(action,
+                                                                                       observation=self.cur_observation,
+                                                                                       llm_config=self.conf.llm_config,
+                                                                                       **kwargs)
+            reward = 1
+        except Exception as e:
+            fail_error = str(e)
+
+        info = {"exception": fail_error}
+        terminated = kwargs.get("terminated", False)
+        for res in action_result:
+            if res.is_done:
+                terminated = res.is_done
+                info['done'] = True
+                self._finish = True
+            if res.error:
+                fail_error += res.error
+
+        contains_write_to_file = any(act.action_name == BrowserAction.WRITE_TO_FILE.value.name for act in action if act)
+        if contains_write_to_file:
+            msg = ""
+            for action_result_elem in action_result:
+                msg = action_result_elem.content
+            # write_to_file observation
+            return (Observation(content=msg, action_result=action_result, info=info),
+                    reward,
+                    terminated,
+                    kwargs.get("truncated", False),
+                    info)
+        elif fail_error:
+            # failed error observation
+            return (Observation(action_result=action_result, observer=self.name()),
+                    reward,
+                    terminated,
+                    kwargs.get("truncated", False),
+                    info)
+        else:
+            # normal observation
+            observation = await self._get_observation(info)
+            observation.action_result = action_result
+            observation.ability = action[-1].action_name
+            self.cur_observation = observation
+            return (observation,
+                    reward,
+                    terminated,
+                    kwargs.get("truncated", False),
+                    info)

chapter9/gaia-experience/AWorld/examples/common/tools/browsers/browser.py

@@ -0,0 +1,368 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+import base64
+import json
+import os
+import subprocess
+import time
+import traceback
+from importlib import resources
+from pathlib import Path
+from typing import Any, Dict, Tuple, List, Union
+
+from aworld.config import ConfigDict
+from examples.common.tools.common import package
+from examples.common.tools.tool_action import BrowserAction
+from aworld.core.common import Observation, ActionModel, ActionResult
+from aworld.logs.util import logger
+from aworld.core.tool.base import action_executor, ToolFactory
+from aworld.core.tool.base import Tool
+from aworld.utils.import_package import is_package_installed
+from examples.common.tools.browsers.action.executor import BrowserToolActionExecutor
+from examples.common.tools.browsers.util.dom import DomTree
+from examples.common.tools.conf import BrowserToolConfig
+from examples.common.tools.browsers.util.dom_build import build_dom_tree
+from aworld.utils import import_package
+from aworld.tools.utils import build_observation
+
+URL_MAX_LENGTH = 4096
+UTF8 = "".join(chr(x) for x in range(0, 55290))
+ASCII = "".join(chr(x) for x in range(32, 128))
+
+BROWSER = "browser"
+
+
+@ToolFactory.register(name=BROWSER,
+                      desc="browser",
+                      supported_action=BrowserAction,
+                      conf_file_name=f'browser_tool.yaml',
+                      dir=f"{Path(__file__).parent.absolute()}")
+class BrowserTool(Tool):
+    def __init__(self, conf: Union[ConfigDict, BrowserToolConfig], **kwargs) -> None:
+        super(BrowserTool, self).__init__(conf, **kwargs)
+
+        self.initialized = False
+        self._finish = False
+        self.record_trace = self.conf.get("enable_recording", False)
+        self.sleep_after_init = self.conf.get("sleep_after_init", False)
+
+        dom_js_path = self.conf.get('dom_js_path')
+        if dom_js_path and os.path.exists(dom_js_path):
+            with open(dom_js_path, 'r') as read:
+                self.js_code = read.read()
+        else:
+            self.js_code = resources.read_text(f'{package}.browsers.script',
+                                               'buildDomTree.js')
+        self.cur_observation = None
+        if not is_package_installed('playwright'):
+            import_package("playwright")
+            logger.info("playwright install...")
+            try:
+                subprocess.check_call('playwright install', shell=True, timeout=300)
+            except Exception as e:
+                logger.error(f"Fail to auto execute playwright install, you can install manually\n {e}")
+
+    def init(self) -> None:
+        from playwright.sync_api import sync_playwright
+
+        if self.initialized:
+            return
+
+        self.context_manager = sync_playwright()
+        self.playwright = self.context_manager.start()
+        self.browser = self._create_browser()
+        self.browser_context = self._create_browser_context()
+
+        if self.record_trace:
+            self.browser_context.tracing.start(screenshots=True, snapshots=True)
+
+        self.page = self.browser_context.new_page()
+        if self.conf.get("custom_executor"):
+            self.action_executor = BrowserToolActionExecutor(self)
+        else:
+            self.action_executor = action_executor
+        self.initialized = True
+
+    def _create_browser(self):
+        browse_name = self.conf.get("browse_name", "chromium")
+        browse = getattr(self.playwright, browse_name)
+        cdp_url = self.conf.get("cdp_url")
+        wss_url = self.conf.get("wss_url")
+        if cdp_url:
+            if browse_name != "chromium":
+                logger.warning(f"{browse_name} unsupported CDP, will use chromium browser")
+                browse = self.playwright.chromium
+            logger.info(f"Connecting to remote browser via CDP {cdp_url}")
+            browser = browse.connect_over_cdp(cdp_url)
+        elif wss_url:
+            logger.info(f"Connecting to remote browser via wss {wss_url}")
+            browser = browse.connect(wss_url)
+        else:
+            headless = self.conf.get("headless", False)
+            slow_mo = self.conf.get("slow_mo", 0)
+            disable_security_args = []
+            if self.conf.get('disable_security', False):
+                disable_security_args = ['--disable-web-security',
+                                         '--disable-site-isolation-trials',
+                                         '--disable-features=IsolateOrigins,site-per-process']
+            args = ['--no-sandbox',
+                    '--disable-crash-reporte',
+                    '--disable-blink-features=AutomationControlled',
+                    '--disable-infobars',
+                    '--disable-background-timer-throttling',
+                    '--disable-popup-blocking',
+                    '--disable-backgrounding-occluded-windows',
+                    '--disable-renderer-backgrounding',
+                    '--disable-window-activation',
+                    '--disable-focus-on-load',
+                    '--no-first-run',
+                    '--no-default-browser-check',
+                    '--no-startup-window',
+                    '--window-position=0,0',
+                    '--window-size=1280,720'] + disable_security_args
+            browser = browse.launch(
+                headless=headless,
+                slow_mo=slow_mo,
+                args=args,
+                proxy=self.conf.get('proxy'),
+            )
+        return browser
+
+    def _create_browser_context(self):
+        """Creates a new browser context with anti-detection measures and loads cookies if available."""
+        from playwright.sync_api import ViewportSize
+
+        browser = self.browser
+        if self.conf.get("cdp_url") and len(browser.contexts) > 0:
+            context = browser.contexts[0]
+        else:
+            viewport_size = ViewportSize(width=self.conf.get("width", 1280),
+                                         height=self.conf.get("height", 720))
+            disable_security = self.conf.get('disable_security', False)
+
+            context = browser.new_context(viewport=viewport_size,
+                                          no_viewport=False,
+                                          user_agent=self.conf.get('user_agent'),
+                                          java_script_enabled=True,
+                                          bypass_csp=disable_security,
+                                          ignore_https_errors=disable_security,
+                                          record_video_dir=self.conf.get('working_dir'),
+                                          record_video_size=viewport_size,
+                                          locale=self.conf.get('locale'),
+                                          storage_state=self.conf.get("storage_state", None),
+                                          geolocation=self.conf.get("geolocation", None),
+                                          device_scale_factor=1)
+            if "chromium" == self.conf.get("browse_name", "chromium"):
+                context.grant_permissions(['camera', 'microphone'])
+
+        if self.conf.get('working_dir'):
+            context.tracing.start(screenshots=True, snapshots=True, sources=True)
+
+        cookie_file = self.conf.get('cookies_file')
+        if cookie_file and os.path.exists(cookie_file):
+            with open(cookie_file, 'r') as read:
+                cookies = json.loads(read.read())
+                context.add_cookies(cookies)
+                logger.info(f'Cookies load from {cookie_file} finished')
+
+        if self.conf.get('private'):
+            js = resources.read_text(f"{package}.browsers.script", "stealth.min.js")
+            context.add_init_script(js)
+
+        return context
+
+    def get_cur_page(self):
+        return self.page
+
+    def screenshot(self, full_page: bool = False) -> str:
+        """Returns a base64 encoded screenshot of the current page.
+
+        Args:
+            full_page: When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport.
+
+        Returns:
+            Base64 of the page screenshot
+        """
+        page = self.get_cur_page()
+
+        try:
+            page.bring_to_front()
+            page.wait_for_load_state(timeout=2000)
+        except:
+            logger.warning("bring to front load timeout")
+            pass
+
+        screenshot = page.screenshot(
+            full_page=full_page,
+            animations='disabled',
+            timeout=600000
+        )
+        logger.info("page screenshot finished")
+        screenshot_base64 = base64.b64encode(screenshot).decode('utf-8')
+        return screenshot_base64
+
+    def _get_observation(self, info: Dict[str, Any] = {}) -> Observation:
+        fail_error = info.get('exception')
+        if fail_error:
+            return Observation(observer=self.name(), action_result=[ActionResult(error=fail_error)])
+
+        try:
+            dom_tree = self._parse_dom_tree()
+            image = self.screenshot()
+            pixels_above, pixels_below = self._scroll_info()
+            info.update({"pixels_above": pixels_above,
+                         "pixels_below": pixels_below,
+                         "url": self.page.url})
+            return Observation(observer=self.name(),
+                               dom_tree=dom_tree,
+                               image=image,
+                               info=info)
+        except Exception as e:
+            try:
+                self.page.go_back()
+            except:
+                logger.warning("current page abnormal, new page to use.")
+                self.page = self.browser_context.new_page()
+            try:
+                dom_tree = self._parse_dom_tree()
+                image = self.screenshot()
+                pixels_above, pixels_below = self._scroll_info()
+                info.update({"pixels_above": pixels_above,
+                             "pixels_below": pixels_below,
+                             "url": self.page.url})
+                return Observation(observer=self.name(), dom_tree=dom_tree, image=image, info=info)
+            except Exception as e:
+                logger.warning(f"build observation fail, {traceback.format_exc()}")
+                return Observation(observer=self.name(), action_result=[ActionResult(error=traceback.format_exc())])
+
+    def _parse_dom_tree(self) -> DomTree:
+        args = {
+            'doHighlightElements': self.conf.get("do_highlight", True),
+            'focusHighlightIndex': self.conf.get("focus_highlight", -1),
+            'viewportExpansion': self.conf.get("viewport_expansion", 0),
+            'debugMode': logger.getEffectiveLevel() == 10,
+        }
+        element_tree, element_map = build_dom_tree(self.page, self.js_code, args)
+        return DomTree(element_tree=element_tree, element_map=element_map)
+
+    def _scroll_info(self) -> tuple[int, int]:
+        """Get scroll position information for the current page."""
+        scroll_y = self.page.evaluate('window.scrollY')
+        viewport_height = self.page.evaluate('window.innerHeight')
+        total_height = self.page.evaluate('document.documentElement.scrollHeight')
+        pixels_above = scroll_y
+        pixels_below = total_height - (scroll_y + viewport_height)
+        return pixels_above, pixels_below
+
+    def reset(self, *, seed: int | None = None, options: Dict[str, str] | None = None) -> Tuple[
+        Observation, Dict[str, Any]]:
+        super().reset(seed=seed, options=options)
+        if self.initialized:
+            observation = self._get_observation()
+            observation.action_result = [ActionResult(content='start', keep=True)]
+            self.cur_observation = observation
+            return observation, {}
+
+        self.close()
+        self.init()
+
+        if self.sleep_after_init > 0:
+            time.sleep(self.sleep_after_init)
+
+        observation = self._get_observation()
+        observation.action_result = [ActionResult(content='start', keep=True)]
+        self.cur_observation = observation
+        return observation, {}
+
+    @property
+    def finished(self) -> bool:
+        return self._finish
+
+    def save_trace(self, trace_path: str | Path) -> None:
+        if self.record_trace:
+            self.browser_context.tracing.stop(path=trace_path)
+
+    def close(self) -> None:
+        if hasattr(self, 'context') and self.browser_context:
+            self.browser_context.close()
+        if hasattr(self, 'browser') and self.browser:
+            self.browser.close()
+        if hasattr(self, 'playwright') and self.playwright:
+            self.playwright.stop()
+
+        if self.initialized:
+            self.context_manager.__exit__()
+
+    def do_step(self, action: List[ActionModel], **kwargs) -> Tuple[
+        Observation, float, bool, bool, Dict[str, Any]]:
+        if not self.initialized:
+            raise RuntimeError("Call init first before calling step.")
+
+        if not action:
+            logger.warning(f"{self.name()} has no action")
+            return build_observation(observer=self.name(), ability='', content='no action'), 0., False, False, {}
+
+        reward = 0
+        fail_error = ""
+        action_result = None
+
+        invalid_acts: List[int] = []
+        for i, act in enumerate(action):
+            if act.tool_name != BROWSER:
+                logger.warning(f"tool {act.tool_name} is not a browser!")
+                invalid_acts.append(i)
+
+        if invalid_acts:
+            for i in invalid_acts:
+                action[i] = None
+
+        try:
+            action_result, self.page = self.action_executor.execute_action(action,
+                                                                           observation=self.cur_observation,
+                                                                           llm_config=self.conf.llm_config,
+                                                                           **kwargs)
+            reward = 1
+        except Exception as e:
+            fail_error = str(e)
+
+        info = {"exception": fail_error}
+        terminated = kwargs.get("terminated", False)
+        if action_result:
+            for res in action_result:
+                if res.is_done:
+                    terminated = res.is_done
+                    info['done'] = True
+                    self._finish = True
+                if res.error:
+                    fail_error += res.error
+
+        contains_write_to_file = any(act.action_name == BrowserAction.WRITE_TO_FILE.value.name for act in action if act)
+        if contains_write_to_file:
+            msg = ""
+            for action_result_elem in action_result:
+                msg = action_result_elem.content
+            # write_to_file observation
+            return (Observation(content=msg, action_result=action_result, info=info),
+                    reward,
+                    terminated,
+                    kwargs.get("truncated", False),
+                    info)
+        elif fail_error:
+            # failed error observation
+            return (Observation(action_result=action_result, observer=self.name()),
+                    reward,
+                    terminated,
+                    kwargs.get("truncated", False),
+                    info)
+        else:
+            # normal observation
+            observation = self._get_observation(info)
+            observation.ability = action[-1].action_name
+            observation.action_result = action_result
+            self.cur_observation = observation
+            return (observation,
+                    reward,
+                    terminated,
+                    kwargs.get("truncated", False),
+                    info)

chapter9/gaia-experience/AWorld/examples/common/tools/browsers/browser_tool.yaml

@@ -0,0 +1,24 @@
+browse_name: chromium
+headless: False
+width: 1280
+height: 720
+slow_mo: 0
+disable_security: False
+custom_executor: False
+dom_js_path: 
+private: 
+locale: 
+geolocation: 
+storage_state: 
+do_highlight: True
+focus_highlight: -1
+viewport_expansion: 0
+cdp_url: 
+wss_url: 
+proxy: 
+cookies_file: 
+working_dir: 
+enable_recording: False
+sleep_after_init: 0
+max_retry: 3
+reuse: True

chapter9/gaia-experience/AWorld/examples/common/tools/browsers/requirements.txt

@@ -0,0 +1,2 @@
+playwright
+markdownify

chapter9/gaia-experience/AWorld/examples/common/tools/browsers/util/dom.py

@@ -0,0 +1,210 @@
+# coding: utf-8
+
+from dataclasses import dataclass
+from typing import Optional, Dict, List
+
+from pydantic import BaseModel
+
+
+class Coordinates(BaseModel):
+    x: int
+    y: int
+
+
+class CoordinateSet(BaseModel):
+    top_left: Coordinates
+    top_right: Coordinates
+    bottom_left: Coordinates
+    bottom_right: Coordinates
+    center: Coordinates
+    width: int
+    height: int
+
+
+class ViewportInfo(BaseModel):
+    width: int
+    height: int
+
+
+@dataclass
+class HashedDomElement:
+    """
+    Hash of the dom element to be used as a unique identifier
+    """
+
+    branch_path_hash: str
+    attributes_hash: str
+    xpath_hash: str
+
+
+@dataclass(frozen=False)
+class DOMBaseNode:
+    is_visible: bool
+    # Use None as default and set parent later to avoid circular reference issues
+    parent: Optional['DOMElementNode']
+
+
+@dataclass(frozen=False)
+class DOMTextNode(DOMBaseNode):
+    text: str
+    type: str = 'TEXT_NODE'
+
+    def has_parent_with_highlight_index(self) -> bool:
+        current = self.parent
+        while current is not None:
+            # stop if the element has a highlight index (will be handled separately)
+            if current.highlight_index is not None:
+                return True
+
+            current = current.parent
+        return False
+
+    def is_parent_in_viewport(self) -> bool:
+        if self.parent is None:
+            return False
+        return self.parent.is_in_viewport
+
+    def is_parent_top_element(self) -> bool:
+        if self.parent is None:
+            return False
+        return self.parent.is_top_element
+
+
+@dataclass(frozen=False)
+class DOMElementNode(DOMBaseNode):
+    """
+    xpath: the xpath of the element from the last root node (shadow root or iframe OR document if no shadow root or iframe).
+    To properly reference the element we need to recursively switch the root node until we find the element (work you way up the tree with `.parent`)
+    """
+
+    tag_name: str
+    xpath: str
+    attributes: Dict[str, str]
+    children: List[DOMBaseNode]
+    is_interactive: bool = False
+    is_top_element: bool = False
+    is_in_viewport: bool = False
+    shadow_root: bool = False
+    highlight_index: Optional[int] = None
+    viewport_coordinates: Optional[CoordinateSet] = None
+    page_coordinates: Optional[CoordinateSet] = None
+    viewport_info: Optional[ViewportInfo] = None
+
+    def __repr__(self) -> str:
+        tag_str = f'<{self.tag_name}'
+
+        # Add attributes
+        for key, value in self.attributes.items():
+            tag_str += f' {key}="{value}"'
+        tag_str += '>'
+
+        # Add extra info
+        extras = []
+        if self.is_interactive:
+            extras.append('interactive')
+        if self.is_top_element:
+            extras.append('top')
+        if self.shadow_root:
+            extras.append('shadow-root')
+        if self.highlight_index is not None:
+            extras.append(f'highlight:{self.highlight_index}')
+        if self.is_in_viewport:
+            extras.append('in-viewport')
+
+        if extras:
+            tag_str += f' [{", ".join(extras)}]'
+
+        return tag_str
+
+    def get_all_text_till_next_clickable_element(self, max_depth: int = -1) -> str:
+        text_parts = []
+
+        def collect_text(node: DOMBaseNode, current_depth: int) -> None:
+            if max_depth != -1 and current_depth > max_depth:
+                return
+
+            # Skip this branch if we hit a highlighted element (except for the current node)
+            if isinstance(node, DOMElementNode) and node != self and node.highlight_index is not None:
+                return
+
+            if isinstance(node, DOMTextNode):
+                text_parts.append(node.text)
+            elif isinstance(node, DOMElementNode):
+                for child in node.children:
+                    collect_text(child, current_depth + 1)
+
+        collect_text(self, 0)
+        return '\n'.join(text_parts).strip()
+
+    def clickable_elements_to_string(self, include_attributes: list[str] | None = None) -> str:
+        """Convert the processed DOM content to HTML."""
+        formatted_text = []
+
+        def process_node(node: DOMBaseNode, depth: int) -> None:
+            if isinstance(node, DOMElementNode):
+                # Add element with highlight_index
+                if node.highlight_index is not None:
+                    attributes_str = ''
+                    text = node.get_all_text_till_next_clickable_element()
+                    if include_attributes:
+                        attributes = list(
+                            set(
+                                [
+                                    str(value)
+                                    for key, value in node.attributes.items()
+                                    if key in include_attributes and value != node.tag_name
+                                ]
+                            )
+                        )
+                        if text in attributes:
+                            attributes.remove(text)
+                        attributes_str = ';'.join(attributes)
+                    line = f'[{node.highlight_index}]<{node.tag_name} '
+                    if attributes_str:
+                        line += f'{attributes_str}'
+                    if text:
+                        if attributes_str:
+                            line += f'>{text}'
+                        else:
+                            line += f'{text}'
+                    line += '/>'
+                    formatted_text.append(line)
+
+                # Process children regardless
+                for child in node.children:
+                    process_node(child, depth + 1)
+
+            elif isinstance(node, DOMTextNode):
+                # Add text only if it doesn't have a highlighted parent
+                if not node.has_parent_with_highlight_index() and node.is_visible:  # and node.is_parent_top_element()
+                    formatted_text.append(f'{node.text}')
+
+        process_node(self, 0)
+        return '\n'.join(formatted_text)
+
+    def get_file_upload_element(self, check_siblings: bool = True) -> Optional['DOMElementNode']:
+        # Check if current element is a file input
+        if self.tag_name == 'input' and self.attributes.get('type') == 'file':
+            return self
+
+        # Check children
+        for child in self.children:
+            if isinstance(child, DOMElementNode):
+                result = child.get_file_upload_element(check_siblings=False)
+                if result:
+                    return result
+
+        # Check siblings only for the initial call
+        if check_siblings and self.parent:
+            for sibling in self.parent.children:
+                if sibling is not self and isinstance(sibling, DOMElementNode):
+                    result = sibling.get_file_upload_element(check_siblings=False)
+                    if result:
+                        return result
+
+        return None
+
+
+class DomTree(BaseModel):
+    element_tree: DOMElementNode
+    element_map: Dict[int, DOMElementNode]

chapter9/gaia-experience/AWorld/examples/common/tools/browsers/util/dom_build.py

@@ -0,0 +1,138 @@
+# coding: utf-8
+
+# Derived from browser_use DomService, we use it as a utility method, and supports sync and async.
+
+import gc
+import json
+
+from typing import Dict, Any, Tuple, Optional
+
+from aworld.utils.async_func import async_func
+from examples.common.tools.browsers.util.dom import DOMElementNode, DOMBaseNode, DOMTextNode, ViewportInfo
+from aworld.logs.util import logger
+
+
+async def async_build_dom_tree(page, js_code: str, args: Dict[str, Any]) -> Tuple[DOMElementNode, Dict[int, DOMElementNode]]:
+    if await page.evaluate('1+1') != 2:
+        raise ValueError('The page cannot evaluate javascript code properly')
+
+    # NOTE: We execute JS code in the browser to extract important DOM information.
+    #       The returned hash map contains information about the DOM tree and the
+    #       relationship between the DOM elements.
+    try:
+        eval_page = await page.evaluate(js_code, args)
+    except Exception as e:
+        logger.error('Error evaluating JavaScript: %s', e)
+        raise
+
+    # Only log performance metrics in debug mode
+    if args.get("debugMode") and 'perfMetrics' in eval_page:
+        logger.debug('DOM Tree Building Performance Metrics:\n%s', json.dumps(eval_page['perfMetrics'], indent=2))
+
+    return await async_func(_construct_dom_tree)(eval_page)
+
+
+def build_dom_tree(page, js_code: str, args: Dict[str, Any]) -> Tuple[DOMElementNode, Dict[int, DOMElementNode]]:
+    if page.evaluate('1+1') != 2:
+        raise ValueError('The page cannot evaluate javascript code properly')
+
+    # NOTE: We execute JS code in the browser to extract important DOM information.
+    #       The returned hash map contains information about the DOM tree and the
+    #       relationship between the DOM elements.
+    try:
+        eval_page = page.evaluate(js_code, args)
+    except Exception as e:
+        logger.error('Error evaluating JavaScript: %s', e)
+        raise
+
+    # Only log performance metrics in debug mode
+    if args.get("debugMode") and 'perfMetrics' in eval_page:
+        logger.debug('DOM Tree Building Performance Metrics:\n%s', json.dumps(eval_page['perfMetrics'], indent=2))
+
+    return _construct_dom_tree(eval_page)
+
+
+def _construct_dom_tree(eval_page: dict, ) -> tuple[DOMElementNode, Dict[int, DOMElementNode]]:
+    js_node_map = eval_page['map']
+    js_root_id = eval_page['rootId']
+
+    selector_map = {}
+    node_map = {}
+
+    for id, node_data in js_node_map.items():
+        node, children_ids = _parse_node(node_data)
+        if node is None:
+            continue
+
+        node_map[id] = node
+
+        if isinstance(node, DOMElementNode) and node.highlight_index is not None:
+            selector_map[node.highlight_index] = node
+
+        # NOTE: We know that we are building the tree bottom up
+        #       and all children are already processed.
+        if isinstance(node, DOMElementNode):
+            for child_id in children_ids:
+                if child_id not in node_map:
+                    continue
+
+                child_node = node_map[child_id]
+
+                child_node.parent = node
+                node.children.append(child_node)
+
+    html_to_dict = node_map[str(js_root_id)]
+
+    del node_map
+    del js_node_map
+    del js_root_id
+
+    gc.collect()
+
+    if html_to_dict is None or not isinstance(html_to_dict, DOMElementNode):
+        raise ValueError('Failed to parse HTML to dictionary')
+
+    return html_to_dict, selector_map
+
+
+def _parse_node(node_data: dict, ) -> Tuple[Optional[DOMBaseNode], list[int]]:
+    if not node_data:
+        return None, []
+
+    # Process text nodes immediately
+    if node_data.get('type') == 'TEXT_NODE':
+        text_node = DOMTextNode(
+            text=node_data['text'],
+            is_visible=node_data['isVisible'],
+            parent=None,
+        )
+        return text_node, []
+
+    # Process coordinates if they exist for element nodes
+
+    viewport_info = None
+
+    if 'viewport' in node_data:
+        viewport_info = ViewportInfo(
+            width=node_data['viewport']['width'],
+            height=node_data['viewport']['height'],
+        )
+
+    element_node = DOMElementNode(
+        tag_name=node_data['tagName'],
+        xpath=node_data['xpath'],
+        attributes=node_data.get('attributes', {}),
+        children=[],
+        is_visible=node_data.get('isVisible', False),
+        is_interactive=node_data.get('isInteractive', False),
+        is_top_element=node_data.get('isTopElement', False),
+        is_in_viewport=node_data.get('isInViewport', False),
+        highlight_index=node_data.get('highlightIndex'),
+        shadow_root=node_data.get('shadowRoot', False),
+        parent=None,
+        viewport_info=viewport_info,
+    )
+
+    children_ids = node_data.get('children', [])
+
+    return element_node, children_ids

chapter9/gaia-experience/AWorld/examples/common/tools/common.py

@@ -0,0 +1,37 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+from enum import Enum
+
+package = 'examples.common.tools'
+
+
+class Tools(Enum):
+    """Tool list supported in the framework, pre-defined to avoid spelling errors."""
+    BROWSER = "browser"
+    ANDROID = "android"
+    GYM = "openai_gym"
+    SEARCH_API = "search_api"
+    SHELL = "shell"
+    PYTHON_EXECUTE = "python_execute"
+    CODE_EXECUTE = "code_execute"
+    FILE = "file"
+    IMAGE_ANALYSIS = "image_analysis"
+    DOCUMENT_ANALYSIS = "document_analysis"
+    HTML = "html"
+    MCP = "mcp"
+
+
+class Agents(Enum):
+    """Agent supported in the framework, pre-defined to avoid spelling errors."""
+    BROWSER = "browser_agent"
+    ANDROID = "android_agent"
+    SEARCH = "search_agent"
+    CODE_EXECUTE = "code_execute_agent"
+    FILE = "file_agent"
+    IMAGE_ANALYSIS = "image_analysis_agent"
+    SHELL = "shell_agent"
+    DOCUMENT = "document_agent"
+    GYM = "gym_agent"
+    PLAN = "plan_agent"
+    EXECUTE = "execute_agent"
+    SUMMARY = "summary_agent"

chapter9/gaia-experience/AWorld/examples/common/tools/conf.py

@@ -0,0 +1,43 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+import os
+
+from aworld.config.conf import ToolConfig, ModelConfig
+
+
+class BrowserToolConfig(ToolConfig):
+    headless: bool = False
+    keep_browser_open: bool = True
+    private: bool = True
+    browse_name: str = "chromium"
+    custom_executor: bool = False
+    width: int = 1280
+    height: int = 720
+    slow_mo: int = 0
+    disable_security: bool = False
+    dom_js_path: str = None
+    locale: str = None
+    geolocation: str = None
+    storage_state: str = None
+    do_highlight: bool = True
+    focus_highlight: int = -1
+    viewport_expansion: int = 0
+    cdp_url: str = None
+    wss_url: str = None
+    proxy: str = None
+    cookies_file: str = None
+    working_dir: str = None
+    enable_recording: bool = False
+    sleep_after_init: float = 0
+    max_retry: int = 3
+    llm_config: ModelConfig = ModelConfig()
+    max_extract_content_input_tokens: int = 64000
+    max_extract_content_output_tokens: int = 5000
+    reuse: bool = True
+
+
+class AndroidToolConfig(ToolConfig):
+    avd_name: str | None = None
+    adb_path: str | None = os.path.expanduser('~') + "/Library/Android/sdk/platform-tools/adb"
+    emulator_path: str | None = os.path.expanduser('~') + "/Library/Android/sdk/emulator/emulator"
+    headless: bool | None = None

chapter9/gaia-experience/AWorld/examples/common/tools/document/__init__.py

@@ -0,0 +1,2 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.

chapter9/gaia-experience/AWorld/examples/common/tools/document/actions.py

@@ -0,0 +1,12 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+from examples.common.tools.tool_action import DocumentExecuteAction
+from aworld.core.tool.action_factory import ActionFactory
+from aworld.core.tool.action import ExecutableAction
+
+
+@ActionFactory.register(name=DocumentExecuteAction.DOCUMENT_ANALYSIS.value.name,
+                        desc=DocumentExecuteAction.DOCUMENT_ANALYSIS.value.desc,
+                        tool_name="document_analysis")
+class ExecuteAction(ExecutableAction):
+    """Only one action, define it, implemented can be omitted. Act in tool."""

chapter9/gaia-experience/AWorld/examples/common/tools/document/document.py

@@ -0,0 +1,529 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+import json
+import os
+import base64
+import tempfile
+import subprocess
+from pathlib import Path
+from typing import Any, Dict, Tuple
+from urllib.parse import urlparse
+
+from pydantic import BaseModel
+
+from aworld.config import ToolConfig
+from examples.common.tools.tool_action import DocumentExecuteAction
+from aworld.core.common import Observation, ActionModel, ActionResult
+from aworld.core.tool.base import ToolFactory, Tool
+from aworld.logs.util import logger
+from examples.common.tools.document.utils import encode_image_from_file, encode_image_from_url
+from aworld.utils import import_package, import_packages
+from aworld.tools.utils import build_observation
+
+
+class InputDocument(BaseModel):
+    document_path: str | None = None
+
+
+@ToolFactory.register(name="document_analysis",
+                      desc="document analysis",
+                      supported_action=DocumentExecuteAction,
+                      conf_file_name=f'document_analysis_tool.yaml',
+                      dir=f"{Path(__file__).parent.absolute()}")
+class DocumentTool(Tool):
+    def __init__(self, conf: ToolConfig, **kwargs) -> None:
+        """Init document tool."""
+        import_package('cv2', install_name='opencv-python')
+        import_packages(['xmltodict', 'pandas', 'docx2markdown', 'PyPDF2', 'numpy'])
+        super(DocumentTool, self).__init__(conf, **kwargs)
+        self.cur_observation = None
+        self.content = None
+        self.keyframes = []
+        self.init()
+        self.step_finished = True
+
+    def reset(self, *, seed: int | None = None, options: Dict[str, str] | None = None) -> Tuple[
+        Observation, dict[str, Any]]:
+        super().reset(seed=seed, options=options)
+
+        self.close()
+        self.step_finished = True
+        return build_observation(observer=self.name(),
+                                 ability=DocumentExecuteAction.DOCUMENT_ANALYSIS.value.name), {}
+
+    def init(self) -> None:
+        self.initialized = True
+
+    def close(self) -> None:
+        pass
+
+    def finished(self) -> bool:
+        return self.step_finished
+
+    def do_step(self, actions: list[ActionModel], **kwargs) -> Tuple[Observation, float, bool, bool, Dict[str, Any]]:
+        self.step_finished = False
+        reward = 0.
+        fail_error = ""
+        observation = build_observation(observer=self.name(),
+                                        ability=DocumentExecuteAction.DOCUMENT_ANALYSIS.value.name)
+        info = {}
+        try:
+            if not actions:
+                raise ValueError("actions is empty")
+            action = actions[0]
+            document_path = action.params.get("document_path", "")
+            if not document_path:
+                raise ValueError("document path invalid")
+            output, keyframes, error = self.document_analysis(document_path)
+            observation.content = output
+            observation.action_result.append(
+                ActionResult(is_done=True,
+                             success=False if error else True,
+                             content=f"{output}",
+                             error=f"{error}",
+                             keep=False))
+            info['key_frame'] = f"{keyframes}"
+            reward = 1.
+        except Exception as e:
+            fail_error = str(e)
+        finally:
+            self.step_finished = True
+        info["exception"] = fail_error
+        info.update(kwargs)
+        return (observation, reward, kwargs.get("terminated", False),
+                kwargs.get("truncated", False), info)
+
+    def document_analysis(self, document_path):
+        import xmltodict
+        error = None
+        # Initialize content to empty list to avoid None return
+        self.content = []
+        try:
+            if any(document_path.endswith(ext) for ext in [".jpg", ".jpeg", ".png"]):
+                parsed_url = urlparse(document_path)
+                is_url = all([parsed_url.scheme, parsed_url.netloc])
+                if not is_url:
+                    base64_image = encode_image_from_file(document_path)
+                else:
+                    base64_image = encode_image_from_url(document_path)
+                self.content = f"data:image/jpeg;base64,{base64_image}"
+
+            if any(document_path.endswith(ext) for ext in ["xls", "xlsx"]):
+                try:
+                    try:
+                        import pandas as pd
+                    except ImportError:
+                        error = "pandas library not found. Please install pandas: pip install pandas"
+                        return self.content, self.keyframes, error
+
+                    excel_data = {}
+
+                    with pd.ExcelFile(document_path) as xls:
+                        sheet_names = xls.sheet_names
+                        for sheet_name in sheet_names:
+                            df = pd.read_excel(xls, sheet_name=sheet_name)
+                            sheet_data = df.to_dict(orient='records')
+                            excel_data[sheet_name] = sheet_data
+
+                    self.content = json.dumps(excel_data, ensure_ascii=False)
+                    logger.info(f"Successfully processed Excel file: {document_path}")
+                    logger.info(f"Found {len(sheet_names)} sheets: {', '.join(sheet_names)}")
+
+                except Exception as excel_error:
+                    error = str(excel_error)
+
+            if any(document_path.endswith(ext) for ext in ["json", "jsonl", "jsonld"]):
+                with open(document_path, "r", encoding="utf-8") as f:
+                    self.content = json.load(f)
+                f.close()
+
+            if any(document_path.endswith(ext) for ext in ["xml"]):
+                data = None
+                with open(document_path, "r", encoding="utf-8") as f:
+                    data = f.read()
+                f.close()
+
+                try:
+                    self.content = xmltodict.parse(data)
+                    logger.info(f"The extracted xml data is: {self.content}")
+
+                except Exception as e:
+                    logger.info(f"The raw xml data is: {data}")
+                    error = str(e)
+                    self.content = data
+
+            if any(document_path.endswith(ext) for ext in ["doc", "docx"]):
+                from docx2markdown._docx_to_markdown import docx_to_markdown
+                file_name = os.path.basename(document_path)
+                md_file_path = f"{file_name}.md"
+                docx_to_markdown(document_path, md_file_path)
+                with open(md_file_path, "r") as f:
+                    self.content = f.read()
+                f.close()
+
+            if any(document_path.endswith(ext) for ext in ["pdf"]):
+                # try using pypdf to extract text from pdf
+                try:
+                    from PyPDF2 import PdfReader
+
+                    # Open file in binary mode for PdfReader
+                    f = open(document_path, "rb")
+                    reader = PdfReader(f)
+                    extracted_text = ""
+                    for page in reader.pages:
+                        extracted_text += page.extract_text()
+                    self.content = extracted_text
+                    f.close()
+                except Exception as pdf_error:
+                    error = str(pdf_error)
+
+            # audio
+            if any(document_path.endswith(ext.lower()) for ext in [".mp3", ".wav", ".wave"]):
+                try:
+                    # audio-> base64
+                    with open(document_path, "rb") as audio_file:
+                        audio_bytes = audio_file.read()
+                        audio_base64 = base64.b64encode(audio_bytes).decode('utf-8')
+
+                    # ext
+                    ext = os.path.splitext(document_path)[1].lower()
+                    mime_type = "audio/mpeg" if ext == ".mp3" else "audio/wav"
+
+                    # data URI
+                    self.content = f"data:{mime_type};base64,{audio_base64}"
+                except Exception as audio_error:
+                    error = str(audio_error)
+                    logger.error(f"Error processing audio file: {error}")
+
+            # video
+            if any(document_path.endswith(ext.lower()) for ext in [".mp4", ".avi", ".mov", ".mkv", ".flv", ".wmv"]):
+                try:
+                    try:
+                        import cv2
+                        import numpy as np
+                    except ImportError:
+                        error = "Required libraries not found. Please install opencv-python: pip install opencv-python"
+                        return None, None, error
+
+                    # create temp dir
+                    temp_dir = tempfile.mkdtemp()
+
+                    # 1.get audio -> base64
+                    audio_path = os.path.join(temp_dir, "extracted_audio.mp3")
+
+                    #  get audio by ffmpeg
+                    try:
+                        subprocess.run([
+                            "ffmpeg", "-i", document_path, "-q:a", "0",
+                            "-map", "a", audio_path, "-y"
+                        ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+                        # audio->base64
+                        with open(audio_path, "rb") as audio_file:
+                            audio_bytes = audio_file.read()
+                            audio_base64 = base64.b64encode(audio_bytes).decode('utf-8')
+
+                        audio_data_uri = f"data:audio/mpeg;base64,{audio_base64}"
+                    except (subprocess.SubprocessError, FileNotFoundError) as e:
+                        logger.warning(f"Failed to extract audio: {str(e)}")
+                        audio_data_uri = None
+
+                    # 2. get keyframes
+                    cap = cv2.VideoCapture(document_path)
+
+                    if not cap.isOpened():
+                        raise ValueError(f"Could not open video file: {document_path}")
+
+                    # get video message
+                    fps = cap.get(cv2.CAP_PROP_FPS)
+                    frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+                    duration = frame_count / fps if fps > 0 else 0
+
+                    # keyframes policy- per duration/10s，max 10
+                    keyframes_count = min(10, int(frame_count))
+                    frames_interval = max(1, int(frame_count / keyframes_count))
+
+                    self.keyframes = []
+                    frame_index = 0
+
+                    while True:
+                        ret, frame = cap.read()
+                        if not ret:
+                            break
+
+                        # per frames_interval save
+                        if frame_index % frames_interval == 0:
+                            # save JPEG -> base64
+                            _, buffer = cv2.imencode(".jpg", frame)
+                            img_base64 = base64.b64encode(buffer).decode('utf-8')
+                            time_position = frame_index / fps if fps > 0 else 0
+
+                            self.keyframes.append(f"data:image/jpeg;base64,{img_base64}")
+
+                            if len(self.keyframes) >= keyframes_count:
+                                break
+
+                        frame_index += 1
+
+                    cap.release()
+
+                    self.content = audio_data_uri
+                    logger.info(f"Successfully processed video file: {document_path}")
+                    logger.info(f"Extracted {len(self.keyframes)} keyframes and audio track")
+                    # clean tmp files
+                    try:
+                        os.remove(audio_path)
+                        os.rmdir(temp_dir)
+                    except Exception as cleanup_error:
+                        logger.warning(f"Error cleaning up temp files: {str(cleanup_error)}")
+
+                except Exception as video_error:
+                    error = str(video_error)
+                    logger.error(f"Error processing video file: {error}")
+
+            if any(document_path.endswith(ext) for ext in ["pptx"]):
+                try:
+                    # Initialize content list and empty keyframes
+                    self.content = []
+                    self.keyframes = []
+
+                    # Check if file exists
+                    if not os.path.exists(document_path):
+                        error = f"File does not exist: {document_path}"
+                        return self.content, self.keyframes, error
+
+                    # Check if file is readable
+                    if not os.access(document_path, os.R_OK):
+                        error = f"File is not readable: {document_path}"
+                        return self.content, self.keyframes, error
+
+                    # Check file size
+                    try:
+                        file_size = os.path.getsize(document_path)
+
+                        if file_size == 0:
+                            error = "File is empty"
+                            return self.content, self.keyframes, error
+                    except Exception as size_error:
+                        logger.warning(f"Cannot get file size: {str(size_error)}")
+
+                    try:
+                        # Import required libraries
+                        from pptx import Presentation
+                        from PIL import Image, ImageDraw, ImageFont
+                        import io
+                    except ImportError as import_error:
+                        error = f"Missing required libraries: {str(import_error)}. Please install: pip install python-pptx Pillow"
+                        return self.content, self.keyframes, error
+
+                    # Create temporary directory for images
+                    try:
+                        temp_dir = tempfile.mkdtemp()
+                    except Exception as temp_dir_error:
+                        error = f"Failed to create temporary directory: {str(temp_dir_error)}"
+                        return self.content, self.keyframes, error
+
+                    # Open presentation
+                    try:
+                        presentation = Presentation(document_path)
+                        # Get total slides count
+                        total_slides = len(presentation.slides)
+
+                        if total_slides == 0:
+                            error = "PPTX file does not contain any slides"
+                            return self.content, self.keyframes, error
+
+                        # Process each slide
+                        for i, slide in enumerate(presentation.slides):
+
+                            # Generate temporary file path for current slide
+                            img_path = os.path.join(temp_dir, f"slide_{i + 1}.jpg")
+
+                            # Get slide dimensions
+                            try:
+                                slide_width = presentation.slide_width
+                                slide_height = presentation.slide_height
+
+                                # PPTX dimensions are in EMU (English Metric Unit)
+                                # 1 inch = 914400 EMU, 1 cm = 360000 EMU
+                                # Convert to pixels (assuming 96 DPI)
+                                slide_width_px = int(slide_width / 914400 * 96 * 10)
+                                slide_height_px = int(slide_height / 914400 * 96 * 10)
+
+                                # Ensure dimensions are reasonable positive integers
+                                slide_width_px = max(1, min(slide_width_px, 4000))  # Limit max width to 4000px
+                                slide_height_px = max(1, min(slide_height_px, 3000))  # Limit max height to 3000px
+
+                            except Exception as size_error:
+                                # Use default dimensions
+                                slide_width_px = 960  # Default width 960px
+                                slide_height_px = 720  # Default height 720px
+
+                            # Create blank image
+                            try:
+                                # Log operation start
+
+                                # Create blank image
+                                try:
+                                    slide_img = Image.new('RGB', (slide_width_px, slide_height_px), 'white')
+                                    draw = ImageDraw.Draw(slide_img)
+                                except Exception as img_create_error:
+                                    logger.error(
+                                        f"Slide {i + 1} blank image creation failed: {str(img_create_error) or 'Unknown error'}")
+                                    raise
+
+                                # Draw slide number
+                                try:
+                                    font = ImageFont.load_default()
+                                    draw.text((20, 20), f"Slide {i + 1}/{total_slides}", fill="black", font=font)
+                                except Exception as font_error:
+                                    logger.warning(f"Failed to draw slide number: {str(font_error) or 'Unknown error'}")
+
+                                # Record shape count
+                                try:
+                                    shape_count = len(slide.shapes)
+                                except Exception as shape_count_error:
+                                    logger.warning(
+                                        f"Failed to get slide {i + 1} shape count: {str(shape_count_error) or 'Unknown error'}")
+                                    shape_count = 0
+
+                                # Try to render shapes on image
+                                shape_success_count = 0
+                                shape_fail_count = 0
+
+                                try:
+                                    for j, shape in enumerate(slide.shapes):
+                                        try:
+                                            shape_type = type(shape).__name__
+
+                                            # Process images
+                                            if hasattr(shape, 'image') and shape.image:
+                                                try:
+                                                    # Extract image from shape
+                                                    image_stream = io.BytesIO(shape.image.blob)
+                                                    img = Image.open(image_stream)
+
+                                                    # Calculate position
+                                                    left = shape.left
+                                                    top = shape.top
+
+                                                    # Paste image onto slide
+                                                    slide_img.paste(img, (left, top))
+                                                    shape_success_count += 1
+                                                except Exception as img_error:
+                                                    logger.warning(
+                                                        f"Failed to process image {j + 1} in slide {i + 1}: {str(img_error) or 'Unknown error'}")
+                                                    if not str(img_error):
+                                                        import traceback
+                                                        logger.warning(
+                                                            f"Image processing stack: {traceback.format_exc()}")
+                                                    shape_fail_count += 1
+
+                                            # Process text
+                                            elif hasattr(shape, 'text') and shape.text:
+                                                try:
+                                                    text = shape.text[:30] + "..." if len(
+                                                        shape.text) > 30 else shape.text
+                                                    # Simple text rendering
+                                                    text_left = shape.left
+                                                    text_top = shape.top
+                                                    draw.text((text_left, text_top), shape.text, fill="black",
+                                                              font=font)
+                                                    shape_success_count += 1
+                                                except Exception as text_error:
+                                                    logger.warning(
+                                                        f"Failed to process text {j + 1} in slide {i + 1}: {str(text_error) or 'Unknown error'}")
+                                                    if not str(text_error):
+                                                        import traceback
+                                                        logger.warning(
+                                                            f"Text processing stack: {traceback.format_exc()}")
+                                                    shape_fail_count += 1
+                                            else:
+                                                logger.info(
+                                                    f"Shape {j + 1} in slide {i + 1} is neither image nor text, skipping")
+                                        except Exception as shape_error:
+                                            if not str(shape_error):
+                                                import traceback
+                                                logger.warning(f"Shape processing stack: {traceback.format_exc()}")
+                                            shape_fail_count += 1
+                                except Exception as shapes_iteration_error:
+                                    logger.error(
+                                        f"Failed while iterating through shapes in slide {i + 1}: {str(shapes_iteration_error) or 'Unknown error'}")
+                                    if not str(shapes_iteration_error):
+                                        import traceback
+                                        logger.error(f"Shape iteration stack: {traceback.format_exc()}")
+
+                                # Save slide image
+                                try:
+                                    slide_img.save(img_path, 'JPEG')
+
+                                    # Check if image was saved successfully
+                                    if not os.path.exists(img_path):
+                                        raise ValueError(f"Saved image file does not exist: {img_path}")
+
+                                    file_size = os.path.getsize(img_path)
+                                    if file_size == 0:
+                                        raise ValueError(
+                                            f"Saved image file is empty: {img_path}, size: {file_size} bytes")
+
+                                    # Convert to base64
+                                    try:
+                                        base64_image = encode_image_from_file(img_path)
+                                        self.content.append(f"data:image/jpeg;base64,{base64_image}")
+                                    except Exception as base64_error:
+                                        error_msg = str(base64_error) or "Unknown base64 conversion error"
+                                        if not str(base64_error):
+                                            import traceback
+                                            logger.error(f"Base64 conversion stack: {traceback.format_exc()}")
+                                        raise ValueError(f"Base64 conversion error: {error_msg}")
+
+                                except Exception as save_error:
+                                    error_msg = str(save_error) or "Unknown save error"
+                                    logger.error(f"Failed to save slide {i + 1} as image: {error_msg}")
+                                    if not str(save_error):
+                                        import traceback
+                                        logger.error(f"Image save stack: {traceback.format_exc()}")
+                                    raise ValueError(f"Image save error: {error_msg}")
+
+                            except Exception as slide_render_error:
+                                error_msg = str(slide_render_error) or "Unknown rendering error"
+                                logger.error(f"Failed to render slide {i + 1}: {error_msg}")
+                                if not str(slide_render_error):
+                                    import traceback
+                                    logger.error(f"Slide rendering stack: {traceback.format_exc()}")
+                                # Continue processing next slide, don't interrupt the entire process
+                                continue
+
+                    except Exception as pptx_error:
+                        error = f"Failed to process PPTX file: {str(pptx_error)}"
+                        import traceback
+
+                    # Clean up temporary files
+                    try:
+                        for file in os.listdir(temp_dir):
+                            try:
+                                file_path = os.path.join(temp_dir, file)
+                                os.remove(file_path)
+                            except Exception as file_error:
+                                logger.warning(f"Failed to delete temporary file: {str(file_error)}")
+                        os.rmdir(temp_dir)
+                    except Exception as cleanup_error:
+                        logger.warning(f"Failed to clean up temporary files: {str(cleanup_error)}")
+
+                    if len(self.content) > 0:
+                        logger.info(f"Extracted {len(self.content)} slides")
+                    else:
+                        error = error or "Could not extract any slides from PPTX file"
+                        logger.error(error)
+
+                except Exception as outer_error:
+                    error = f"Error occurred during PPTX file processing: {str(outer_error)}"
+                    import traceback
+
+                return self.content, self.keyframes, error
+
+        finally:
+            pass
+
+        return self.content, self.keyframes, error

chapter9/gaia-experience/AWorld/examples/common/tools/document/document_analysis_tool.yaml

@@ -0,0 +1,4 @@
+custom_executor: False
+enable_recording: False
+working_dir:
+max_retry: 3

chapter9/gaia-experience/AWorld/examples/common/tools/document/utils.py

@@ -0,0 +1,32 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+import base64
+from io import BytesIO
+
+
+def encode_image_from_url(image_url):
+    from aworld.utils.import_package import import_package
+    import_package("requests")
+
+    import requests
+    from PIL import Image
+
+    response = requests.get(image_url)
+    image = Image.open(BytesIO(response.content))
+
+    max_size = 1024
+    if max(image.size) > max_size:
+        ratio = max_size / max(image.size)
+        new_size = (int(image.size[0] * ratio), int(image.size[1] * ratio))
+        image = image.resize(new_size, Image.LANCZOS)
+
+    buffered = BytesIO()
+    image_format = image.format if image.format else 'JPEG'
+    image.save(buffered, format=image_format)
+    img_str = base64.b64encode(buffered.getvalue()).decode()
+    return img_str
+
+
+def encode_image_from_file(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode()

chapter9/gaia-experience/AWorld/examples/common/tools/gym_tool/__init__.py

@@ -0,0 +1,2 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.

chapter9/gaia-experience/AWorld/examples/common/tools/gym_tool/actions.py

@@ -0,0 +1,12 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+from examples.common.tools.tool_action import GymAction
+from aworld.core.tool.action_factory import ActionFactory
+from aworld.core.tool.action import ExecutableAction
+
+
+@ActionFactory.register(name=GymAction.PLAY.value.name,
+                        desc=GymAction.PLAY.value.desc,
+                        tool_name="openai_gym")
+class Play(ExecutableAction):
+    """"""

chapter9/gaia-experience/AWorld/examples/common/tools/gym_tool/async_openai_gym.py

@@ -0,0 +1,158 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+from pathlib import Path
+from typing import Dict, Any, Tuple, SupportsFloat, Union, List
+
+from pydantic import BaseModel
+
+from aworld.config import ConfigDict
+from examples.common.tools.tool_action import GymAction
+from aworld.core.common import ActionModel, Observation, ActionResult
+from aworld.core.tool.base import AsyncTool, ToolFactory
+from aworld.utils.import_package import import_packages
+from aworld.tools.utils import build_observation
+
+
+class ActionType(object):
+    DISCRETE = 'discrete'
+    CONTINUOUS = 'continuous'
+
+
+@ToolFactory.register(name="openai_gym",
+                      desc="gym classic control game",
+                      asyn=True,
+                      supported_action=GymAction,
+                      conf_file_name=f'openai_gym_tool.yaml',
+                      dir=f"{Path(__file__).parent.absolute()}")
+class OpenAIGym(AsyncTool):
+    def __init__(self, conf: Union[Dict[str, Any], ConfigDict, BaseModel], **kwargs) -> None:
+        """Gym environment constructor.
+
+        Args:
+            env_id: gym environment full name
+            wrappers: gym environment wrapper list
+        """
+        import_packages(['pygame', 'gymnasium'])
+        super(OpenAIGym, self).__init__(conf, **kwargs)
+        self.env_id = self.conf.get("env_id")
+        self._render = self.conf.get('render', True)
+        if self._render:
+            kwargs['render_mode'] = self.conf.get('render_mode', True)
+        kwargs.pop('name', None)
+        self.env = self._gym_env_wrappers(self.env_id, self.conf.get("wrappers", []), **kwargs)
+        self.action_space = self.env.action_space
+
+    async def do_step(self, actions: List[ActionModel], **kwargs) -> Tuple[
+        Observation, SupportsFloat, bool, bool, Dict[str, Any]]:
+        if self._render:
+            await self.render()
+        action = actions[0].params['result']
+        action = OpenAIGym.transform_action(action=action)
+        state, reward, terminal, truncate, info = self.env.step(action)
+        info.update(kwargs)
+        self._finished = terminal
+
+        action_results = []
+        for _ in actions:
+            action_results.append(ActionResult(content=OpenAIGym.transform_state(state=state), success=True))
+        return (build_observation(observer=self.name(),
+                                  action_result=action_results,
+                                  ability=GymAction.PLAY.value.name,
+                                  content=OpenAIGym.transform_state(state=state),
+                                  env_id=self.env_id,
+                                  done=terminal,
+                                  **kwargs),
+                reward,
+                terminal,
+                truncate,
+                info)
+
+    async def render(self):
+        return self.env.render()
+
+    async def close(self):
+        if self.env:
+            self.env.close()
+        self.env = None
+
+    async def reset(self, *, seed: int | None = None, options: Dict[str, str] | None = None) -> Tuple[
+        Any, Dict[str, Any]]:
+        state = self.env.reset()
+        return build_observation(observer=self.name(),
+                                 ability=GymAction.PLAY.value.name,
+                                 content=OpenAIGym.transform_state(state=state),
+                                 env_id=self.env_id,
+                                 done=False), {}
+
+    def _action_dim(self):
+        from gymnasium import spaces
+
+        if isinstance(self.env.action_space, spaces.Discrete):
+            self.action_type = ActionType.DISCRETE
+            return self.env.action_space.n
+        elif isinstance(self.env.action_space, spaces.Box):
+            self.action_type = ActionType.CONTINUOUS
+            return self.env.action_space.shape[0]
+        else:
+            raise Exception('unsupported env.action_space: {}'.format(self.env.action_space))
+
+    def _state_dim(self):
+        if len(self.env.observation_space.shape) == 1:
+            return self.env.observation_space.shape[0]
+        else:
+            raise Exception('unsupported observation_space.shape: {}'.format(self.env.observation_space))
+
+    def _gym_env_wrappers(self, env_id, wrappers: list = [], **kwargs):
+        import gymnasium
+
+        env = gymnasium.make(env_id, **kwargs)
+
+        if wrappers:
+            for wrapper in wrappers:
+                env = wrapper(env)
+
+        return env
+
+    @staticmethod
+    def transform_state(state: Any):
+        if isinstance(state, tuple):
+            states = dict()
+            for n, state in enumerate(state):
+                state = OpenAIGym.transform_state(state=state)
+                if isinstance(state, dict):
+                    for name, state in state.items():
+                        states['gym{}-{}'.format(n, name)] = state
+                else:
+                    states['gym{}'.format(n)] = state
+            return states
+        elif isinstance(state, dict):
+            states = dict()
+            for state_name, state in state.items():
+                state = OpenAIGym.transform_state(state=state)
+                if isinstance(state, dict):
+                    for name, state in state.items():
+                        states['{}-{}'.format(state_name, name)] = state
+                else:
+                    states['{}'.format(state_name)] = state
+            return states
+        else:
+            return state
+
+    @staticmethod
+    def transform_action(action: Any):
+        if not isinstance(action, dict):
+            return action
+        else:
+            actions = dict()
+            for name, action in action.items():
+                if '-' in name:
+                    name, inner_name = name.split('-', 1)
+                    if name not in actions:
+                        actions[name] = dict()
+                    actions[name][inner_name] = action
+                else:
+                    actions[name] = action
+            for name, action in actions.items():
+                if isinstance(action, dict):
+                    actions[name] = OpenAIGym.transform_action(action=action)
+            return actions

chapter9/gaia-experience/AWorld/examples/common/tools/gym_tool/openai_gym.py

@@ -0,0 +1,154 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+from pathlib import Path
+from typing import Dict, Any, Tuple, SupportsFloat, List, Union
+
+from aworld.config import ConfigDict, ToolConfig
+from examples.common.tools.tool_action import GymAction
+from aworld.core.common import Observation, ActionModel, ActionResult
+from aworld.core.tool.base import Tool, ToolFactory
+from aworld.utils.import_package import import_packages
+from aworld.tools.utils import build_observation
+
+
+class ActionType(object):
+    DISCRETE = 'discrete'
+    CONTINUOUS = 'continuous'
+
+
+@ToolFactory.register(name="openai_gym",
+                      desc="gym classic control game",
+                      supported_action=GymAction,
+                      conf_file_name=f'openai_gym_tool.yaml',
+                      dir=f"{Path(__file__).parent.absolute()}")
+class OpenAIGym(Tool):
+    def __init__(self, conf: Union[Dict[str, Any], ConfigDict, ToolConfig], **kwargs) -> None:
+        """Gym environment constructor.
+
+        Args:
+            env_id: gym environment full name
+            wrappers: gym environment wrapper list
+        """
+        import_packages(['pygame', 'gymnasium'])
+        super(OpenAIGym, self).__init__(conf, **kwargs)
+        self.env_id = self.conf.get("env_id")
+        self._render = self.conf.get('render', True)
+        if self._render:
+            kwargs['render_mode'] = self.conf.get('render_mode', 'human')
+        kwargs.pop('name', None)
+        self.env = self._gym_env_wrappers(self.env_id, self.conf.get("wrappers", []), **kwargs)
+        self.action_space = self.env.action_space
+
+    def do_step(self, actions: List[ActionModel], **kwargs) -> Tuple[
+        Observation, SupportsFloat, bool, bool, Dict[str, Any]]:
+        if self._render:
+            self.render()
+        action = actions[0].params['result']
+        action = OpenAIGym.transform_action(action=action)
+        state, reward, terminal, truncate, info = self.env.step(action)
+        info.update(kwargs)
+        self._finished = terminal
+
+        action_results = []
+        for _ in actions:
+            action_results.append(ActionResult(content=OpenAIGym.transform_state(state=state), success=True))
+        return (build_observation(observer=self.name(),
+                                  action_result=action_results,
+                                  ability=GymAction.PLAY.value.name,
+                                  content=OpenAIGym.transform_state(state=state),
+                                  env_id=self.env_id,
+                                  done=terminal,
+                                  **kwargs),
+                reward,
+                terminal,
+                truncate,
+                info)
+
+    def render(self):
+        return self.env.render()
+
+    def close(self):
+        if self.env:
+            self.env.close()
+        self.env = None
+
+    def reset(self, *, seed: int | None = None, options: Dict[str, str] | None = None) -> Tuple[Any, Dict[str, Any]]:
+        state = self.env.reset()
+        return build_observation(observer=self.name(),
+                                 ability=GymAction.PLAY.value.name,
+                                 content=OpenAIGym.transform_state(state=state),
+                                 env_id=self.env_id,
+                                 done=False), {}
+
+    def _action_dim(self):
+        from gymnasium import spaces
+
+        if isinstance(self.env.action_space, spaces.Discrete):
+            self.action_type = ActionType.DISCRETE
+            return self.env.action_space.n
+        elif isinstance(self.env.action_space, spaces.Box):
+            self.action_type = ActionType.CONTINUOUS
+            return self.env.action_space.shape[0]
+        else:
+            raise Exception('unsupported env.action_space: {}'.format(self.env.action_space))
+
+    def _state_dim(self):
+        if len(self.env.observation_space.shape) == 1:
+            return self.env.observation_space.shape[0]
+        else:
+            raise Exception('unsupported observation_space.shape: {}'.format(self.env.observation_space))
+
+    def _gym_env_wrappers(self, env_id, wrappers: list = [], **kwargs):
+        import gymnasium
+
+        env = gymnasium.make(env_id, **kwargs)
+
+        if wrappers:
+            for wrapper in wrappers:
+                env = wrapper(env)
+
+        return env
+
+    @staticmethod
+    def transform_state(state: Any):
+        if isinstance(state, tuple):
+            states = dict()
+            for n, state in enumerate(state):
+                state = OpenAIGym.transform_state(state=state)
+                if isinstance(state, dict):
+                    for name, state in state.items():
+                        states['gym{}-{}'.format(n, name)] = state
+                else:
+                    states['gym{}'.format(n)] = state
+            return states
+        elif isinstance(state, dict):
+            states = dict()
+            for state_name, state in state.items():
+                state = OpenAIGym.transform_state(state=state)
+                if isinstance(state, dict):
+                    for name, state in state.items():
+                        states['{}-{}'.format(state_name, name)] = state
+                else:
+                    states['{}'.format(state_name)] = state
+            return states
+        else:
+            return state
+
+    @staticmethod
+    def transform_action(action: Any):
+        if not isinstance(action, dict):
+            return action
+        else:
+            actions = dict()
+            for name, action in action.items():
+                if '-' in name:
+                    name, inner_name = name.split('-', 1)
+                    if name not in actions:
+                        actions[name] = dict()
+                    actions[name][inner_name] = action
+                else:
+                    actions[name] = action
+            for name, action in actions.items():
+                if isinstance(action, dict):
+                    actions[name] = OpenAIGym.transform_action(action=action)
+            return actions

chapter9/gaia-experience/AWorld/examples/common/tools/gym_tool/openai_gym_tool.yaml

@@ -0,0 +1,3 @@
+env_id: "CartPole-v1"
+render_mode: "human"
+render: True

chapter9/gaia-experience/AWorld/examples/common/tools/gym_tool/requirements.txt

@@ -0,0 +1,2 @@
+gymnasium~=1.1.0
+pygame~=2.6.1

chapter9/gaia-experience/AWorld/examples/common/tools/html/__init__.py

@@ -0,0 +1,2 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.

chapter9/gaia-experience/AWorld/examples/common/tools/html/actions.py

@@ -0,0 +1,68 @@
+# coding: utf-8
+
+import os
+import re
+
+from typing import Tuple, Any
+
+from aworld.core.tool.action_factory import ActionFactory
+from aworld.core.common import ActionModel, ActionResult
+from aworld.logs.util import logger
+from aworld.core.tool.action import ExecutableAction
+from aworld.models.llm import get_llm_model, call_llm_model
+
+
+@ActionFactory.register(name="write_html",
+                        desc="a tool use for write html.",
+                        tool_name="html")
+class WriteHTML(ExecutableAction):
+    def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
+        logger.info("start write html!")
+        goal = action.params.get("goal")
+        information = action.params.get("information")
+
+        llm_conf = kwargs.get("llm_config")
+        llm = get_llm_model(llm_conf)
+        sys_prompt = "you are a helpful html writer."
+        prompt = """Your task is to create a detailed and visually appealing HTML document based on the specified theme. 
+        The document must meet the following requirements, and you should utilize the provided reference materials to ensure accuracy and aesthetic quality.
+        1) HTML Document Requirements
+        Design and write the HTML document according to the following specifications:
+        Theme : {goal}
+        Related Info: {information}
+        Structural Requirements :
+        Use semantic HTML tags (e.g., <header>, <main>, <footer>, <section>) to create a clear and organized structure.
+        Ensure the document includes a header, navigation bar, main content area, and footer.
+        If applicable, add additional sections such as a sidebar, or call-to-action buttons.
+        Styling Requirements :
+        Implement a visually appealing design using CSS, including color schemes, font choices, spacing adjustments, etc.
+        Ensure the page has a responsive layout that works well on different devices (use media queries or frameworks like Bootstrap).
+        Add animations or interactive features (e.g., hover effects on buttons, scroll-triggered animations) to enhance user experience.
+        please give me html code directly, no need other words
+        """
+
+        messages = [{'role': 'system', 'content': sys_prompt},
+                    {'role': 'user', 'content': prompt.format(goal=goal, information=information)}]
+
+        output = call_llm_model(llm,
+                                messages=messages,
+                                model=llm_conf.llm_model_name,
+                                temperature=llm_conf.llm_temperature)
+
+        content = output.content
+        html_pattern = re.compile(r'<html.*?>.*?</html>', re.DOTALL)
+        matches = html_pattern.findall(content)
+
+        title_pattern = re.compile(r'<title.*?>.*?</title>', re.DOTALL)
+        filename = (title_pattern.findall(content)[0]
+                    .replace("<title>", "")
+                    .replace("</title>", "")
+                    .replace(" ", "_") + ".html")
+
+        with open(filename, "a", encoding='utf-8') as f:
+            f.write(matches[0])
+
+        abs_file_path = os.path.abspath(filename)
+        msg = f'Successfully wrote html to {abs_file_path}'
+
+        return ActionResult(content=msg, keep=True, is_done=True), None

chapter9/gaia-experience/AWorld/examples/common/tools/html/html.py

@@ -0,0 +1,10 @@
+# coding: utf-8
+
+from aworld.tools.template_tool import TemplateTool
+from examples.common.tools.tool_action import WriteAction
+from aworld.core.tool.base import ToolFactory
+
+
+@ToolFactory.register(name="html", desc="html tool", supported_action=WriteAction)
+class HtmlTool(TemplateTool):
+    """Html tool"""

chapter9/gaia-experience/AWorld/examples/common/tools/interpreters/actions.py

@@ -0,0 +1,12 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+from examples.common.tools.tool_action import PythonToolAction
+from aworld.core.tool.action_factory import ActionFactory
+from aworld.core.tool.action import ExecutableAction
+
+
+@ActionFactory.register(name=PythonToolAction.EXECUTE.value.name,
+                        desc=PythonToolAction.EXECUTE.value.desc,
+                        tool_name="python_execute")
+class ExecuteAction(ExecutableAction):
+    """Only one action, define it, implemented can be omitted."""

chapter9/gaia-experience/AWorld/examples/common/tools/interpreters/python_execute_tool.yaml

@@ -0,0 +1,5 @@
+custom_executor: False
+enable_recording: False
+working_dir:
+max_retry: 3
+

chapter9/gaia-experience/AWorld/examples/common/tools/interpreters/python_tool.py

@@ -0,0 +1,257 @@
+import sys
+import ast
+import re
+import subprocess
+from pathlib import Path
+from typing import Any, Dict, Tuple, List
+from io import StringIO
+
+from aworld.logs.util import logger
+from aworld.config.conf import ToolConfig
+from examples.common.tools.tool_action import PythonToolAction
+from aworld.core.common import ActionModel, Observation, ActionResult
+from aworld.core.tool.base import Tool, AgentInput, ToolFactory
+from aworld.utils import import_package
+from aworld.tools.utils import build_observation
+
+
+@ToolFactory.register(name="python_execute",
+                      desc="python interpreter tool",
+                      supported_action=PythonToolAction,
+                      conf_file_name=f'python_execute_tool.yaml',
+                      dir=f"{Path(__file__).parent.absolute()}")
+class PythonTool(Tool):
+
+    def __init__(self,
+                 conf: ToolConfig,
+                 **kwargs) -> None:
+        """
+        Initialize the PythonExecutor
+        Args:
+            conf: tool config
+            **kwargs: -
+        Return:
+            None
+        """
+        super(PythonTool, self).__init__(conf, **kwargs)
+        self.type = "function"
+        self.local_namespace = {}
+        self.global_namespace = {}
+        self.original_stdout = sys.stdout
+        self.output_buffer = StringIO()
+        self.installed_packages = set()
+        import_package('langchain_experimental')
+        from langchain_experimental.utilities.python import PythonREPL
+        self.python_repl = PythonREPL()
+
+    def extract_imports(self, code: str) -> set:
+        """
+        Extract import statements
+        Args:
+            code: python code
+        Returns:
+            set: import statements
+        """
+        imports = set()
+
+        try:
+            tree = ast.parse(code)
+
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Import):
+                    # deal import xxx or import xxx as yyy
+                    for name in node.names:
+                        package_name = name.name.split('.')[0]
+                        imports.add(package_name)
+
+                elif isinstance(node, ast.ImportFrom):
+                    # deal from xxx import yyy or from xxx.yyy import zzz
+                    if node.module:
+                        package_name = node.module.split('.')[0]
+                        imports.add(package_name)
+
+        except SyntaxError:
+            import_pattern = r'^import\s+([\w\s,]+)|from\s+(\w+)'
+            for line in code.split('\n'):
+                line = line.strip()
+                match = re.match(import_pattern, line)
+                if match:
+                    if match.group(1):
+
+                        packages = [p.strip() for p in match.group(1).split(',')]
+                        for package in packages:
+                            if package:
+                                package_name = package.split()[0]
+                                imports.add(package_name)
+                    elif match.group(2):
+                        imports.add(match.group(2))
+
+        return imports
+
+    def install_dependencies(self,
+                             packages: set) -> None:
+        """
+        Install dependency packages
+        Args:
+            packages: python third packages
+        Returns:
+            None
+        """
+        for package in packages:
+            try:
+                __import__(package)
+            except ImportError:
+                try:
+                    subprocess.check_call([sys.executable, "-m", "pip", "install", package])
+                    self.installed_packages.add(package)
+                except subprocess.CalledProcessError as e:
+                    logger.warning(f"Failed to install {package}: {str(e)}")
+
+    def uninstall_dependencies(self) -> None:
+        """
+        Uninstall dependency packages
+        Args:
+            -
+        Returns:
+            None
+        """
+        try:
+            for package in self.installed_packages:
+                try:
+                    subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "-y", package])
+                except subprocess.CalledProcessError as e:
+                    logger.warning(f"Failed to uninstall {package}: {str(e)}")
+            self.installed_packages.clear()
+        except Exception as e:
+            logger.warning(f"Failed to uninstall dependencies: {repr(e)}")
+
+    def reset(self,
+              *,
+              seed: int | None = None,
+              options: Dict[str, str] | None = None) -> Tuple[AgentInput, dict[str, Any]]:
+        """
+        Reset the executor
+        Args:
+            seed: -
+            options: -
+        Returns:
+            AgentInput, dict[str, Any]: -
+        """
+        self.close()
+        self.local_namespace = {}
+        self.global_namespace = {}
+        self._finished = False
+        self.installed_packages.clear()
+
+        return build_observation(observer=self.name(),
+                                 ability=PythonToolAction.EXECUTE.value.name), {}
+
+    def close(self) -> None:
+        """
+        Close the executor
+        Returns:
+            None
+        """
+        try:
+            self.uninstall_dependencies()
+            sys.stdout = self.original_stdout
+            self.output_buffer.close()
+            self.local_namespace.clear()
+            self.global_namespace.clear()
+        except:
+            pass
+        finally:
+            self._finished = True
+
+    def do_step(
+            self,
+            actions: List[ActionModel],
+            **kwargs) -> Tuple[Observation, float, bool, bool, dict[str, Any]]:
+        """
+        Step the executor
+        Args:
+            actions: actions
+            **kwargs: -
+        Returns:
+            Observation, float, bool, bool, dict[str, Any]: -
+        """
+        self.step_finished = False
+        reward = 0
+        fail_error = ""
+        observation = build_observation(observer=self.name(),
+                                        ability=PythonToolAction.EXECUTE.value.name)
+        try:
+            if not actions:
+                return (observation, reward,
+                        kwargs.get("terminated",
+                                   False), kwargs.get("truncated", False), {
+                            "exception": "actions is empty"
+                        })
+            for action in actions:
+                code = action.params.get("code", "")
+                if not code:
+                    logger.warning(f"{action} no code to execute.")
+                    continue
+                try:
+                    _, output, error = self.execute(code)
+                    observation.content = output
+                except Exception as e:
+                    error = str(e)
+                    output = error
+
+                observation.action_result.append(
+                    ActionResult(is_done=True,
+                                 success=False if error else True,
+                                 content=f"{output}",
+                                 error=f"{error}",
+                                 keep=False))
+            reward = 1
+        except Exception as e:
+            fail_error = str(e)
+        finally:
+            self._finished = True
+
+        info = {"exception": fail_error}
+        info.update(kwargs)
+        return (observation, reward, kwargs.get("terminated", False),
+                kwargs.get("truncated", False), info)
+
+    def execute(self, code, timeout=300):
+        """
+        Execute the code
+        Args:
+            code: python code
+            timeout: timeout seconds
+        Returns:
+            result, output, error
+        """
+        required_packages = self.extract_imports(code)
+        self.install_dependencies(required_packages)
+        self.python_repl.globals = self.global_namespace
+        self.python_repl.locals = self.local_namespace
+        error = None
+        try:
+            output = self.python_repl.run(code, timeout)
+        except Exception as e:
+            error = f'{repr(e)}'
+        finally:
+            self.uninstall_dependencies()
+        return '', output, error
+
+    def get_execute_result(self):
+        """
+        Get the execute result
+        Returns:
+            output, error
+        """
+        output = None
+        error = ''
+        try:
+            output = self.output_buffer.getvalue()
+            self.output_buffer.truncate(0)
+            self.output_buffer.seek(0)
+            sys.stdout = self.original_stdout
+        except Exception as e:
+            error = f'{repr(e)}'
+            logger.warning(f"Failed to get output, {repr(e)}")
+        return output, error

chapter9/gaia-experience/AWorld/examples/common/tools/tool_action.py

@@ -0,0 +1,324 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+
+from aworld.core.common import ToolActionInfo, ParamInfo
+from aworld.core.tool.action import ToolAction
+
+
+class ChatAction(ToolAction):
+    """chat between agents """
+    TASK_DONE = ToolActionInfo(name="TASK_DONE",
+                               desc="Complete task - with return text and if the task is finished (success=True) or not yet  completly finished (success=False), because last step is reached")
+
+
+class SearchAction(ToolAction):
+    """Info search actions."""
+    WIKI = ToolActionInfo(name="wiki",
+                          input_params={"query": ParamInfo(name="query",
+                                                           type="str",
+                                                           required=True,
+                                                           desc="wiki search query input.")},
+                          desc="Search the entity in WikiPedia and return the summary of the required page, containing factual information about the given entity.")
+    DUCK_GO = ToolActionInfo(name="duck_go",
+                             input_params={"query": ParamInfo(name="query",
+                                                              type="str",
+                                                              required=True,
+                                                              desc="duckduckgo search query input"),
+                                           "source": ParamInfo(name="source",
+                                                               type="str",
+                                                               required=False,
+                                                               desc="duckduckgo search query input.",
+                                                               default_value="text"),
+                                           "max_results": ParamInfo(name="max_results",
+                                                                    type="str",
+                                                                    required=False,
+                                                                    desc="duckduckgo search query input.",
+                                                                    default_value=5)},
+                             desc="Use DuckDuckGo search engine to search information for the given query")
+    GOOGLE = ToolActionInfo(name="google",
+                            input_params={"query": ParamInfo(name="query",
+                                                             type="str",
+                                                             required=True,
+                                                             desc="google search query input."),
+                                          "num_result_pages": ParamInfo(name="num_result_pages",
+                                                                        type="str",
+                                                                        required=False,
+                                                                        desc="google search query input.",
+                                                                        default_value=5)},
+                            desc="Use Google search engine to search information for the given query.")
+    BAIDU = ToolActionInfo(name="baidu",
+                           input_params={"query": ParamInfo(name="query",
+                                                            type="str",
+                                                            required=True,
+                                                            desc="baidu search query input."),
+                                         "num_results": ParamInfo(name="num_results",
+                                                                  type="str",
+                                                                  required=False,
+                                                                  desc="baidu search number of results.",
+                                                                  default_value=5)},
+                           desc="Use Baidu search engine to search information for the given query.")
+
+
+class GymAction(ToolAction):
+    PLAY = ToolActionInfo(name="play",
+                          input_params={"result": ParamInfo(name="result",
+                                                            type="object",
+                                                            required=True,
+                                                            desc="Agent decision result.")},
+                          desc="step")
+
+
+class BrowserAction(ToolAction):
+    """Definition of Browser tool supported action."""
+    GO_TO_URL = ToolActionInfo(name="go_to_url",
+                               input_params={"url": ParamInfo(name="url",
+                                                              type="str",
+                                                              required=True,
+                                                              desc="got to url in page on browser.")},
+                               desc="Navigate to URL in the current tab")
+    INPUT_TEXT = ToolActionInfo(name="input_text",
+                                input_params={"text": ParamInfo(name="text",
+                                                                type="str",
+                                                                required=True,
+                                                                desc="input text in page on browser"),
+                                              "index": ParamInfo(name="index",
+                                                                 type="str",
+                                                                 required=True,
+                                                                 desc="index of click element in page on browser.")},
+                                desc="Input text into a input interactive element")
+    SEARCH = ToolActionInfo(name="search",
+                            input_params={"url": ParamInfo(name="url",
+                                                           type="str",
+                                                           required=True,
+                                                           desc="search url."),
+                                          "query": ParamInfo(name="query",
+                                                             type="str",
+                                                             required=True,
+                                                             desc="search query input in page on browser.")},
+                            desc="Search the query in search engine, Google, Baidu etc., in the current tab, the query should be a search query like humans search in search engine, concrete and not vague or super long. More the single most important items. ")
+    SEARCH_GOOGLE = ToolActionInfo(name="search_google",
+                                   input_params={"url": ParamInfo(name="url",
+                                                                  type="str",
+                                                                  required=True,
+                                                                  desc="search url."),
+                                                 "query": ParamInfo(name="query",
+                                                                    type="str",
+                                                                    required=True,
+                                                                    desc="search query input in google.")},
+                                   desc="Search the query in Google in the current tab, the query should be a search query like humans search in Google, concrete and not vague or super long. More the single most important items. ")
+    GO_BACK = ToolActionInfo(name="go_back",
+                             desc="Go back")
+    SCROLL_DOWN = ToolActionInfo(name="scroll_down",
+                                 input_params={"amount": ParamInfo(name="amount",
+                                                                   type="str",
+                                                                   required=True,
+                                                                   desc="pixel amount.")},
+                                 desc="Scroll down the page by pixel amount - if no amount is specified, scroll down one page")
+    SCROLL_UP = ToolActionInfo(name="scroll_up",
+                               input_params={"amount": ParamInfo(name="amount",
+                                                                 type="str",
+                                                                 required=True,
+                                                                 desc="Pixel amount.")},
+                               desc="Scroll up the page by pixel amount - if no amount is specified, scroll up one page")
+    CLICK_ELEMENT = ToolActionInfo(name="click_element",
+                                   input_params={"index": ParamInfo(name="index",
+                                                                    type="str",
+                                                                    required=True,
+                                                                    desc="Index of click element in page on browser.")},
+                                   desc="Click element")
+    NEW_TAB = ToolActionInfo(name="new_tab",
+                             input_params={"url": ParamInfo(name="url",
+                                                            type="str",
+                                                            required=True,
+                                                            desc="Open url in new tab on browser.")},
+                             desc="Open url in new tab")
+    SWITCH_TAB = ToolActionInfo(name="switch_tab",
+                                input_params={"page_id": ParamInfo(name="page_id",
+                                                                   type="str",
+                                                                   required=True,
+                                                                   desc="Switch tab by page id on browser.")},
+                                desc="Switch tab")
+    WAIT = ToolActionInfo(name="wait",
+                          input_params={"seconds": ParamInfo(name="seconds",
+                                                             type="str",
+                                                             required=True,
+                                                             desc="Wait some seconds.")},
+                          desc="Open url in new tab")
+    EXTRACT_CONTENT = ToolActionInfo(name="extract_content",
+                                     input_params={"goal": ParamInfo(name="goal",
+                                                                     type="str",
+                                                                     required=True,
+                                                                     desc="The goal in page content.")},
+                                     desc="Extract page content to retrieve specific information from the page, e.g. all company names, a specifc description, all information about, links with companies in structured format or simply links")
+    SEND_KEYS = ToolActionInfo(name="send_keys",
+                               input_params={"keys": ParamInfo(name="keys",
+                                                               type="str",
+                                                               required=True,
+                                                               desc="Strings of special keys.")},
+                               desc="Send strings of special keys like Escape,Backspace, Insert, PageDown, Delete, Enter, Shortcuts such as `Control+o`, `Control+Shift+T` are supported as well. This gets used in keyboard.press. ")
+    WRITE_TO_FILE = ToolActionInfo(name="write_to_file",
+                                   input_params={
+                                       "file_path": ParamInfo(
+                                           name="file_path",
+                                           type="str",
+                                           required=False,
+                                           default_value="tmp_result.md",
+                                           desc="Path to the file to write to"
+                                       ),
+                                       "content": ParamInfo(
+                                           name="content",
+                                           type="str",
+                                           required=True,
+                                           desc="Content to write to the file"
+                                       ),
+                                       "mode": ParamInfo(
+                                           name="mode",
+                                           type="str",
+                                           required=False,
+                                           default_value="a",
+                                           desc="File opening mode: 'w' for write (overwrite), 'a' for append (default)"
+                                       )
+                                   },
+                                   desc="Write content to a file")
+    DONE = ToolActionInfo(name="done",
+                          desc="Complete task - with return text and if the task is finished (success=True) or not yet  completly finished (success=False), because last step is reached")
+
+
+class AndroidAction(ToolAction):
+    """Definition of android tool supported action."""
+    TAP = ToolActionInfo(name="tap",
+                         input_params={"tap_index": ParamInfo(name="tap_index",
+                                                              type="str",
+                                                              required=True,
+                                                              desc="Index of tap element.")},
+                         desc="Tap element")
+    SWIPE = ToolActionInfo(name="swipe",
+                           input_params={"index": ParamInfo(name="index",
+                                                            type="str",
+                                                            required=True,
+                                                            desc="Index of swipe the screen."),
+                                         "direction": ParamInfo(name="direction",
+                                                                type="str",
+                                                                required=True,
+                                                                desc="Direction of swipe the screen."),
+                                         "dist": ParamInfo(name="dist",
+                                                           type="str",
+                                                           required=True,
+                                                           desc="Dist of swipe the screen.")},
+                           desc="Swipe the screen")
+    LONG_PRESS = ToolActionInfo(name="long_press",
+                                input_params={"long_press_index": ParamInfo(name="long_press_index",
+                                                                            type="str",
+                                                                            required=True,
+                                                                            desc="Index of the element.")},
+                                desc="Long press the element")
+    INPUT_TEXT = ToolActionInfo(name="input_text",
+                                input_params={"text": ParamInfo(name="text",
+                                                                type="str",
+                                                                required=True,
+                                                                desc="Input text into a input interactive element.")},
+                                desc="Input text into a input interactive element")
+    DONE = ToolActionInfo(name="done",
+                          input_params={"type": ParamInfo(name="type",
+                                                          type="str",
+                                                          required=True,
+                                                          desc="Type of done."),
+                                        "success": ParamInfo(name="success",
+                                                             type="str",
+                                                             required=True,
+                                                             desc="Task success status.")},
+                          desc="task done")
+
+
+class FileAction(ToolAction):
+    """Definition of file supported action."""
+    OPEN = ToolActionInfo(name="open",
+                          input_params={},
+                          desc="")
+
+
+class ImageAnalysisAction(ToolAction):
+    """Definition of image analysis supported action."""
+    ANALYSIS = ToolActionInfo(name="analysis",
+                              input_params={},
+                              desc="")
+
+
+class CodeExecuteAction(ToolAction):
+    """Definition of code execute supported action."""
+    EXECUTE_CODE = ToolActionInfo(
+        name="execute_code",
+        input_params={"code": ParamInfo(name="code",
+                                        type="str",
+                                        required=True,
+                                        desc="The input code to execute. Codes should be complete and runnable (like running a script), and need to explicitly use the print statement to get the output.")},
+        desc="Execute the given codes. Codes should be complete and runnable (like running a script), and need to explicitly use the print statement to get the output.")
+
+
+class ShellAction(ToolAction):
+    """Definition of shell execute supported action."""
+    EXECUTE_SCRIPT = ToolActionInfo(
+        name="execute_script",
+        input_params={"script": ParamInfo(name="script",
+                                          type="str",
+                                          required=True,
+                                          desc="The input script to execute. Script should be complete and runnable, and need to explicitly use the print statement to get the output.")},
+        desc="Execute the given script, need to explicitly use the print statement to get the output.")
+
+
+class DocumentExecuteAction(ToolAction):
+    """Definition of Document execute supported action."""
+    DOCUMENT_ANALYSIS = ToolActionInfo(
+        name="document_analysis",
+        input_params={"document_path": ParamInfo(name="document_path",
+                                                 type="str",
+                                                 required=True,
+                                                 desc="The path of the document to be processed, either a local path or a URL. It can process image, video, audio, ppt, docx, pdf, doc, xls, xlsx and xml, etc.")},
+        desc="Extract the content of a given document (or url) and return the processed text. It can process image, video, audio, ppt, docx, pdf, doc, xls, xlsx and xml, etc. It may filter out some information, resulting in inaccurate content.")
+
+
+class PythonToolAction(ToolAction):
+    """Definition of python code execute supported action."""
+    EXECUTE = ToolActionInfo(
+        name="execute",
+        input_params={"code": ParamInfo(name="code",
+                                        type="str",
+                                        required=True,
+                                        desc="The input python code to execute. Python codes should be complete and runnable (like running a script), and need to explicitly use the print statement to get the output.")},
+        desc="Execute the given python codes. Codes should be complete and runnable (like running a script), and need to explicitly use the print statement to get the output.")
+
+
+class WriteAction(ToolAction):
+    """Info Write actions."""
+    WRITE_HTML = ToolActionInfo(name="write_html",
+                                input_params={"goal": ParamInfo(name="goal",
+                                                                type="str",
+                                                                required=True,
+                                                                desc="the write goal, about theme, requirements for writing html file."),
+                                              "information": ParamInfo(name="information",
+                                                                       type="str",
+                                                                       required=True,
+                                                                       desc="the related information for writing html file. lengths should less than 6000 words."
+                                                                       )
+                                              },
+                                desc="write the html file about `goal` based on `information`.")
+
+class GetTraceAction(ToolAction):
+    """Definition of get trace supported action."""
+    GET_TRACE = ToolActionInfo(
+        name="get_trace",
+        input_params={"trace_id": ParamInfo(name="trace_id",
+                                            type="str",
+                                            required=True,
+                                            desc="The trace id to get.")},
+        desc="Get the trace of the current execution.")
+
+class HumanExecuteAction(ToolAction):
+    """Definition of Human execute supported action."""
+    HUMAN_CONFIRM = ToolActionInfo(
+        name="human_confirm",
+        input_params={"content": ParamInfo(name="content",
+                                                 type="str",
+                                                 required=True,
+                                                 desc="Content for user confirmation")},
+        desc="The main purpose of this tool is to pass given content to the user for confirmation.")

chapter9/gaia-experience/AWorld/examples/common/tools/trace/trace_tool.py

@@ -0,0 +1,160 @@
+import aworld.trace as trace
+import aworld.trace.instrumentation.semconv as semconv
+from aworld.trace.server import get_trace_server
+from aworld.trace.server.util import build_trace_tree
+from aworld.core.tool.base import AsyncTool, AgentInput, ToolFactory
+from examples.common.tools.tool_action import GetTraceAction
+from aworld.tools.utils import build_observation
+from aworld.config.conf import ToolConfig
+from aworld.core.common import Observation, ActionModel, ActionResult
+from typing import Tuple, Dict, Any, List
+from aworld.logs.util import logger
+
+
+@ToolFactory.register(name="trace",
+                      desc="Get the trace of the current execution.",
+                      supported_action=GetTraceAction,
+                      conf_file_name=f'trace_tool.yaml')
+class TraceTool(AsyncTool):
+    def __init__(self,
+                 conf: ToolConfig,
+                 **kwargs) -> None:
+        """
+        Initialize the TraceTool
+        Args:
+            conf: tool config
+            **kwargs: -
+        Return:
+            None
+        """
+        super(TraceTool, self).__init__(conf, **kwargs)
+        self.type = "function"
+        self.get_trace_url = self.conf.get('get_trace_url')
+
+    async def reset(self,
+              *,
+              seed: int | None = None,
+              options: Dict[str, str] | None = None) -> Tuple[AgentInput, dict[str, Any]]:
+        """
+        Reset the executor
+        Args:
+            seed: -
+            options: -
+        Returns:
+            AgentInput, dict[str, Any]: -
+        """
+        self._finished = False
+        return build_observation(observer=self.name(),
+                                 ability=GetTraceAction.GET_TRACE.value.name), {}
+
+    async def close(self) -> None:
+        """
+        Close the executor
+        Returns:
+            None
+        """
+        self._finished = True
+
+    async def do_step(self,
+                actions: List[ActionModel],
+                **kwargs) -> Tuple[Observation, float, bool, bool, dict[str, Any]]:
+        reward = 0
+        fail_error = ""
+        observation = build_observation(observer=self.name(),
+                                        ability=GetTraceAction.GET_TRACE.value.name)
+        results = []
+        try:
+            if not actions:
+                return (observation, reward,
+                        kwargs.get("terminated",
+                                   False), kwargs.get("truncated", False), {
+                            "exception": "actions is empty"
+                        })
+            for action in actions:
+                trace_id = action.params.get("trace_id", "")
+                if not trace_id:
+                    current_span = trace.get_current_span()
+                    if current_span:
+                        trace_id = current_span.get_trace_id()
+                if not trace_id:
+                    logger.warning(f"{action} no trace_id to fetch.")
+                    observation.action_result.append(
+                        ActionResult(is_done=True,
+                                     success=False,
+                                     content="",
+                                     error="no trace_id to fetch",
+                                     keep=False))
+                    continue
+                try:
+                    trace_data = self.fetch_trace_data(trace_id)
+                    # logger.info(f"trace_data={trace_data}")
+                    error = ""
+                except Exception as e:
+                    error = str(e)
+                results.append(trace_data)
+                observation.action_result.append(
+                    ActionResult(is_done=True,
+                                 success=False if error else True,
+                                 content=f"{trace_data}",
+                                 error=f"{error}",
+                                 keep=False))
+
+            observation.content = f"{results}"
+            reward = 1
+        except Exception as e:
+            fail_error = str(e)
+        finally:
+            self._finished = True
+
+        info = {"exception": fail_error}
+        info.update(kwargs)
+        return (observation, reward, kwargs.get("terminated", False),
+                kwargs.get("truncated", False), info)
+
+    def fetch_trace_data(self, trace_id=None):
+        '''
+            fetch trace data from trace server.
+            return trace data, like:
+            {
+                'trace_id': trace_id,
+                'root_span': [],
+            }
+        '''
+        trace_data = {"trace_id": trace_id, "root_span": []}
+        try:
+            if trace_id:
+                trace_server = get_trace_server()
+                if not trace_server:
+                    logger.error("No memory trace server has been set.")
+                else:
+                    trace_storage = trace_server.get_storage()
+                    spans = trace_storage.get_all_spans(trace_id)
+                    if spans:
+                        trace_data["root_span"] = build_trace_tree(spans)
+                        return self.proccess_trace(trace_data)
+            return trace_data
+        except Exception as e:
+            import traceback
+            logger.error(
+                f"Error fetching trace data traceback: {traceback.format_exc()}")
+            return trace_data
+
+    def proccess_trace(self, trace_data):
+        root_spans = trace_data.get("root_span")
+        for span in root_spans:
+            self.choose_attribute(span)
+        return trace_data
+
+    def choose_attribute(self, span):
+        include_attr = [semconv.GEN_AI_USAGE_INPUT_TOKENS,
+                        semconv.GEN_AI_USAGE_OUTPUT_TOKENS, semconv.GEN_AI_USAGE_TOTAL_TOKENS,
+                        semconv.GEN_AI_COMPLETION_TOOL_CALLS, "event.id"]
+        result_attributes = {}
+        origin_attributes = span.get("attributes") or {}
+        for key, value in origin_attributes.items():
+            if key in include_attr:
+                result_attributes[key] = value
+        span["attributes"] = result_attributes
+        if span.get("children"):
+            for child in span.get("children"):
+                self.choose_attribute(child)