ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,824 @@
|
||||
# coding: utf-8
|
||||
# Copyright (c) 2025 inclusionAI.
|
||||
import os
|
||||
import traceback
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Tuple, Any
|
||||
|
||||
from examples.common.tools.tool_action import BrowserAction
|
||||
from aworld.core.tool.action_factory import ActionFactory
|
||||
from aworld.core.common import ActionModel, ActionResult, Observation
|
||||
from examples.common.tools.browsers.util.dom import DOMElementNode
|
||||
from aworld.logs.util import logger
|
||||
from examples.common.tools.browsers.action.utils import DomUtil
|
||||
from aworld.core.tool.action import ExecutableAction
|
||||
from aworld.utils import import_packages
|
||||
from aworld.models.llm import get_llm_model, call_llm_model
|
||||
|
||||
|
||||
def get_page(**kwargs):
|
||||
tool = kwargs.get("tool")
|
||||
if tool is None:
|
||||
page = kwargs.get('page')
|
||||
else:
|
||||
page = tool.page
|
||||
return page
|
||||
|
||||
|
||||
def get_browser(**kwargs):
|
||||
tool = kwargs.get("tool")
|
||||
if tool is None:
|
||||
page = kwargs.get('browser')
|
||||
else:
|
||||
page = tool.context
|
||||
return page
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.GO_TO_URL.value.name,
|
||||
desc=BrowserAction.GO_TO_URL.value.desc,
|
||||
tool_name="browser")
|
||||
class GotoUrl(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.GO_TO_URL.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.GO_TO_URL.name} page is none")
|
||||
return ActionResult(content="no page", keep=True), page
|
||||
|
||||
params = action.params
|
||||
url = params.get("url")
|
||||
if not url:
|
||||
logger.warning("empty url, go to nothing.")
|
||||
return ActionResult(content="empty url", keep=True), page
|
||||
items = url.split('://')
|
||||
if len(items) == 1:
|
||||
if items[0][0] != '/':
|
||||
url = "file://" + os.path.join(os.getcwd(), url)
|
||||
|
||||
page.goto(url)
|
||||
page.wait_for_load_state()
|
||||
msg = f'Navigated to {url}'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.GO_TO_URL.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.GO_TO_URL.name} page is none")
|
||||
return ActionResult(content="no page", keep=True), page
|
||||
|
||||
url = action.params.get("url")
|
||||
if not url:
|
||||
logger.warning("empty url, go to nothing.")
|
||||
return ActionResult(content="empty url", keep=True), page
|
||||
|
||||
items = url.split('://')
|
||||
if len(items) == 1:
|
||||
if items[0][0] != '/':
|
||||
url = "file://" + os.path.join(os.getcwd(), url)
|
||||
|
||||
await page.goto(url)
|
||||
await page.wait_for_load_state()
|
||||
msg = f'Navigated to {url}'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.INPUT_TEXT.value.name,
|
||||
desc=BrowserAction.INPUT_TEXT.value.desc,
|
||||
tool_name="browser")
|
||||
class InputText(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.INPUT_TEXT.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.INPUT_TEXT.name} page is none")
|
||||
return ActionResult(content="input text no page", keep=True), page
|
||||
|
||||
params = action.params
|
||||
index = params.get("index", 0)
|
||||
# compatible with int and str datatype
|
||||
index = int(index)
|
||||
input = params.get("text", "")
|
||||
|
||||
ob: Observation = kwargs.get("observation")
|
||||
if not ob or index not in ob.dom_tree.element_map:
|
||||
raise RuntimeError(f'Element index {index} does not exist')
|
||||
if not input:
|
||||
raise ValueError(f'No input to the page')
|
||||
|
||||
element_node = ob.dom_tree.element_map[index]
|
||||
self.input_to_element(input, page, element_node)
|
||||
msg = f'Input {input} into index {index}'
|
||||
logger.info(f"action {msg}")
|
||||
logger.debug(f'Element xpath: {element_node.xpath}')
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.INPUT_TEXT.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.INPUT_TEXT.name} page is none")
|
||||
return ActionResult(content="input text no page", keep=True), page
|
||||
|
||||
params = action.params
|
||||
index = params.get("index")
|
||||
# compatible with int and str datatype
|
||||
index = int(index)
|
||||
input = params.get("text", "")
|
||||
|
||||
ob: Observation = kwargs.get("observation")
|
||||
if not ob or index not in ob.dom_tree.element_map:
|
||||
raise RuntimeError(f'Element index {index} does not exist')
|
||||
if not input:
|
||||
raise ValueError(f'No input to the page')
|
||||
|
||||
element_node = ob.dom_tree.element_map[index]
|
||||
await self.async_input_to_element(input, page, element_node)
|
||||
msg = f'Input {input} into index {index}'
|
||||
logger.info(f"action {msg}")
|
||||
logger.debug(f'Element xpath: {element_node.xpath}')
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
def input_to_element(self, input: str, page, element_node: DOMElementNode):
|
||||
try:
|
||||
# Highlight before typing
|
||||
# if element_node.highlight_index is not None:
|
||||
# await self._update_state(focus_element=element_node.highlight_index)
|
||||
|
||||
element_handle = DomUtil.get_locate_element(page, element_node)
|
||||
|
||||
if element_handle is None:
|
||||
raise RuntimeError(f'Element: {repr(element_node)} not found')
|
||||
|
||||
# Ensure element is ready for input
|
||||
try:
|
||||
element_handle.wait_for_element_state('stable', timeout=1000)
|
||||
element_handle.scroll_into_view_if_needed(timeout=1000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Get element properties to determine input method
|
||||
is_contenteditable = element_handle.get_property('isContentEditable')
|
||||
|
||||
# Different handling for contenteditable vs input fields
|
||||
if is_contenteditable.json_value():
|
||||
element_handle.evaluate('el => el.textContent = ""')
|
||||
element_handle.type(input, delay=5)
|
||||
else:
|
||||
element_handle.fill(input)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to input text into element: {repr(element_node)}. Error: {str(e)}')
|
||||
raise RuntimeError(f'Failed to input text into index {element_node.highlight_index}')
|
||||
|
||||
async def async_input_to_element(self, input: str, page, element_node: DOMElementNode):
|
||||
try:
|
||||
element_handle = await DomUtil.async_get_locate_element(page, element_node)
|
||||
|
||||
if element_handle is None:
|
||||
raise RuntimeError(f'Element: {repr(element_node)} not found')
|
||||
|
||||
# Ensure element is ready for input
|
||||
try:
|
||||
await element_handle.wait_for_element_state('stable', timeout=1000)
|
||||
await element_handle.scroll_into_view_if_needed(timeout=1000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Get element properties to determine input method
|
||||
is_contenteditable = await element_handle.get_property('isContentEditable')
|
||||
|
||||
# Different handling for contenteditable vs input fields
|
||||
if await is_contenteditable.json_value():
|
||||
await element_handle.evaluate('el => el.textContent = ""')
|
||||
await element_handle.type(input, delay=5)
|
||||
else:
|
||||
await element_handle.fill(input)
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to input text into element: {repr(element_node)}. Error: {str(e)}')
|
||||
raise RuntimeError(f'Failed to input text into index {element_node.highlight_index}')
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.CLICK_ELEMENT.value.name,
|
||||
desc=BrowserAction.CLICK_ELEMENT.value.desc,
|
||||
tool_name="browser")
|
||||
class ClickElement(ExecutableAction):
|
||||
def __init__(self):
|
||||
import_packages(['playwright', 'markdownify'])
|
||||
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
from playwright.sync_api import BrowserContext
|
||||
|
||||
logger.info(f"exec {BrowserAction.CLICK_ELEMENT.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.CLICK_ELEMENT.name} page is none")
|
||||
return ActionResult(content="input text no page", keep=True), page
|
||||
|
||||
browser: BrowserContext = get_browser(**kwargs)
|
||||
if browser is None:
|
||||
logger.warning(f"{BrowserAction.CLICK_ELEMENT.name} browser context is none")
|
||||
return ActionResult(content="none browser context", keep=True), page
|
||||
|
||||
index = action.params.get("index")
|
||||
# compatible with int and str datatype
|
||||
index = int(index)
|
||||
ob: Observation = kwargs.get("observation")
|
||||
if not ob or index not in ob.dom_tree.element_map:
|
||||
raise RuntimeError(f'Element index {index} does not exist')
|
||||
if not input:
|
||||
raise ValueError(f'No input to the page')
|
||||
element_node = ob.dom_tree.element_map[index]
|
||||
|
||||
try:
|
||||
pages = len(browser.pages)
|
||||
msg = f'Clicked button with index {index}: {element_node.get_all_text_till_next_clickable_element(max_depth=2)}'
|
||||
logger.info(msg)
|
||||
|
||||
DomUtil.click_element(page, element_node, browser=browser)
|
||||
logger.debug(f'Element xpath: {element_node.xpath}')
|
||||
if len(browser.pages) > pages:
|
||||
new_tab_msg = 'Open the new tab'
|
||||
msg += f' - {new_tab_msg}'
|
||||
logger.info(new_tab_msg)
|
||||
page = browser.pages[-1]
|
||||
page.bring_to_front()
|
||||
page.wait_for_load_state(timeout=60000)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
except Exception as e:
|
||||
logger.warning(f'Element not clickable with index {index} - most likely the page changed')
|
||||
return ActionResult(error=str(e)), page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.CLICK_ELEMENT.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warn(f"{BrowserAction.CLICK_ELEMENT.name} page is none")
|
||||
return ActionResult(content="input text no page", keep=True), page
|
||||
|
||||
browser = get_browser(**kwargs)
|
||||
if browser is None:
|
||||
logger.warning(f"{BrowserAction.CLICK_ELEMENT.name} browser context is none")
|
||||
return ActionResult(content="none browser context", keep=True), page
|
||||
|
||||
index = action.params.get("index")
|
||||
# compatible with int and str datatype
|
||||
index = int(index)
|
||||
ob: Observation = kwargs.get("observation")
|
||||
if not ob or index not in ob.dom_tree.element_map:
|
||||
raise RuntimeError(f'Element index {index} does not exist')
|
||||
if not input:
|
||||
raise ValueError(f'No input to the page')
|
||||
element_node = ob.dom_tree.element_map[index]
|
||||
pages = len(browser.pages)
|
||||
|
||||
try:
|
||||
await DomUtil.async_click_element(page, element_node, browser=browser)
|
||||
msg = f'Clicked button with index {index}: {element_node.get_all_text_till_next_clickable_element(max_depth=2)}'
|
||||
|
||||
logger.info(msg)
|
||||
logger.debug(f'Element xpath: {element_node.xpath}')
|
||||
if len(browser.pages) > pages:
|
||||
new_tab_msg = 'Open the new tab'
|
||||
msg += f' - {new_tab_msg}'
|
||||
logger.info(new_tab_msg)
|
||||
page = browser.pages[-1]
|
||||
await page.bring_to_front()
|
||||
await page.wait_for_load_state(timeout=60000)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
except Exception as e:
|
||||
logger.warning(f'Element not clickable with index {index} - most likely the page changed')
|
||||
return ActionResult(error=str(e)), page
|
||||
|
||||
|
||||
# SEARCH_ENGINE = {"": "https://www.google.com/search?udm=14&q=",
|
||||
# "google": "https://www.google.com/search?udm=14&q="}
|
||||
|
||||
SEARCH_ENGINE = {"": "https://www.bing.com/search?q=",
|
||||
"google": "https://www.bing.com/search?q="}
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.SEARCH.value.name,
|
||||
desc=BrowserAction.SEARCH.value.desc,
|
||||
tool_name="browser")
|
||||
class Search(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SEARCH.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.SEARCH.name} page is none")
|
||||
return ActionResult(content="search no page", keep=True), page
|
||||
|
||||
params = action.params if action.params else {}
|
||||
engine = params.get("engine", "")
|
||||
url = SEARCH_ENGINE.get(engine)
|
||||
query = params.get("query")
|
||||
page.goto(f'{url}{query}')
|
||||
page.wait_for_load_state()
|
||||
msg = f'Searched for "{query}" in {url}'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SEARCH.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.SEARCH.name} page is none")
|
||||
return ActionResult(content="search no page", keep=True), page
|
||||
|
||||
params = action.params if action.params else {}
|
||||
engine = params.get("engine", "")
|
||||
url = SEARCH_ENGINE.get(engine)
|
||||
query = params.get("query")
|
||||
await page.goto(f'{url}{query}')
|
||||
await page.wait_for_load_state()
|
||||
msg = f'Searched for "{query}" in {url}'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.SEARCH_GOOGLE.value.name,
|
||||
desc=BrowserAction.SEARCH_GOOGLE.value.desc,
|
||||
tool_name="browser")
|
||||
class SearchGoogle(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SEARCH_GOOGLE.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.SEARCH_GOOGLE.name} page is none")
|
||||
return ActionResult(content="search no page", keep=True), page
|
||||
|
||||
query = action.params.get("query")
|
||||
page.goto(f'{SEARCH_ENGINE.get("")}{query}')
|
||||
page.wait_for_load_state()
|
||||
msg = f'Searched for "{query}" in Google'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SEARCH_GOOGLE.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.SEARCH_GOOGLE.name} page is none")
|
||||
return ActionResult(content="search no page", keep=True), page
|
||||
|
||||
query = action.params.get("query")
|
||||
await page.goto(f'{SEARCH_ENGINE.get("")}{query}')
|
||||
await page.wait_for_load_state()
|
||||
msg = f'Searched for "{query}" in Google'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.NEW_TAB.value.name,
|
||||
desc=BrowserAction.NEW_TAB.value.desc,
|
||||
tool_name="browser")
|
||||
class NewTab(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.NEW_TAB.value.name} action")
|
||||
browser = get_browser(**kwargs)
|
||||
url = action.params.get("url")
|
||||
|
||||
new_page = browser.new_page()
|
||||
new_page.wait_for_load_state()
|
||||
|
||||
if url:
|
||||
new_page.goto(url)
|
||||
DomUtil.wait_for_stable_network(new_page)
|
||||
|
||||
msg = f'Opened new tab with {url}'
|
||||
logger.debug(msg)
|
||||
return ActionResult(content=msg, keep=True), new_page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.NEW_TAB.value.name} action")
|
||||
browser = get_browser(**kwargs)
|
||||
url = action.params.get("url")
|
||||
new_page = await browser.new_page()
|
||||
await new_page.wait_for_load_state()
|
||||
|
||||
if url:
|
||||
await new_page.goto(url)
|
||||
DomUtil.wait_for_stable_network(new_page)
|
||||
msg = f'Opened new tab with {url}'
|
||||
logger.debug(msg)
|
||||
return ActionResult(content=msg, keep=True), get_page(**kwargs)
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.GO_BACK.value.name,
|
||||
desc=BrowserAction.GO_BACK.value.desc,
|
||||
tool_name="browser")
|
||||
class GoBack(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.GO_BACK.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.GO_BACK.name} page is none")
|
||||
return ActionResult(content="search no page", keep=True), page
|
||||
|
||||
page.go_back()
|
||||
msg = 'Navigated back'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.GO_BACK.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.GO_BACK.name} page is none")
|
||||
return ActionResult(content="search no page", keep=True), page
|
||||
|
||||
await page.go_back()
|
||||
msg = 'Navigated back'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.EXTRACT_CONTENT.value.name,
|
||||
desc=BrowserAction.EXTRACT_CONTENT.value.desc,
|
||||
tool_name="browser")
|
||||
class ExtractContent(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
import markdownify
|
||||
from langchain_core.prompts import PromptTemplate
|
||||
|
||||
logger.info(f"exec {BrowserAction.EXTRACT_CONTENT.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.EXTRACT_CONTENT.name} page is none")
|
||||
return ActionResult(content="extract content no page", keep=True), page
|
||||
|
||||
goal = action.params.get("goal")
|
||||
llm_config = kwargs.get("llm_config")
|
||||
if llm_config and llm_config.llm_api_key:
|
||||
llm = get_llm_model(llm_config)
|
||||
max_extract_content_output_tokens = kwargs.get("max_extract_content_output_tokens")
|
||||
max_extract_content_input_tokens = kwargs.get("max_extract_content_input_tokens")
|
||||
|
||||
content = markdownify.markdownify(page.content())
|
||||
|
||||
# Truncate content if it exceeds max input tokens
|
||||
if max_extract_content_input_tokens and len(content) > max_extract_content_input_tokens:
|
||||
logger.warning(
|
||||
f"Content length ({len(content)}) exceeds max input tokens ({max_extract_content_input_tokens}). Truncating content.")
|
||||
content = content[:max_extract_content_input_tokens]
|
||||
|
||||
prompt = 'Your task is to extract the content of the page. You will be given a page and a goal and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. Extraction goal: {goal}, Page: {page}'
|
||||
prompt_with_outputlimit = 'Your task is to extract the content of the page. You will be given a page and a goal and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. Extraction goal: {goal}, Page: {page} \n\n#The length of the returned result must be less than {max_extract_content_output_tokens} characters.'
|
||||
template = PromptTemplate(input_variables=['goal', 'page'], template=prompt)
|
||||
|
||||
messages = [{'role': 'user', 'content': template.format(goal=goal, page=content)}]
|
||||
try:
|
||||
output = call_llm_model(llm,
|
||||
messages=messages,
|
||||
model=llm_config.llm_model_name,
|
||||
temperature=llm_config.llm_temperature)
|
||||
result_content = output.content
|
||||
|
||||
# Check if output exceeds the token limit and retry with length-limited prompt if needed
|
||||
if max_extract_content_output_tokens and len(result_content) > max_extract_content_output_tokens:
|
||||
logger.warning(
|
||||
f"Output exceeds maximum length ({len(result_content)} > {max_extract_content_output_tokens}). Retrying with limited prompt.")
|
||||
template_with_limit = PromptTemplate(
|
||||
input_variables=['goal', 'page', 'max_extract_content_output_tokens'],
|
||||
template=prompt_with_outputlimit
|
||||
)
|
||||
messages = [{'role': 'user', 'content': template_with_limit.format(
|
||||
goal=goal,
|
||||
page=content,
|
||||
max_extract_content_output_tokens=max_extract_content_output_tokens,
|
||||
max_tokens=max_extract_content_output_tokens
|
||||
)}]
|
||||
# extract content with length limit
|
||||
output = call_llm_model(llm,
|
||||
messages=messages,
|
||||
model=llm_config.llm_model_name,
|
||||
temperature=llm_config.llm_temperature)
|
||||
result_content = output.content
|
||||
|
||||
msg = f'Extracted from page\n: {result_content}\n'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
except Exception as e:
|
||||
logger.debug(f'Error extracting content: {e}')
|
||||
msg = f'Extracted from page\n: {content}\n'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg), page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
import markdownify
|
||||
from langchain_core.prompts import PromptTemplate
|
||||
|
||||
logger.info(f"exec {BrowserAction.EXTRACT_CONTENT.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.EXTRACT_CONTENT.name} page is none")
|
||||
return ActionResult(content="extract content no page", keep=True), page
|
||||
|
||||
goal = action.params.get("goal")
|
||||
llm_config = kwargs.get("llm_config")
|
||||
if llm_config and llm_config.llm_api_key:
|
||||
llm = get_llm_model(llm_config)
|
||||
content = markdownify.markdownify(await page.content())
|
||||
max_extract_content_output_tokens = kwargs.get("max_extract_content_output_tokens")
|
||||
max_extract_content_input_tokens = kwargs.get("max_extract_content_input_tokens")
|
||||
|
||||
# Truncate content if it exceeds max input tokens
|
||||
if max_extract_content_input_tokens and len(content) > max_extract_content_input_tokens:
|
||||
logger.warning(
|
||||
f"Content length ({len(content)}) exceeds max input tokens ({max_extract_content_input_tokens}). Truncating content.")
|
||||
content = content[:max_extract_content_input_tokens]
|
||||
|
||||
prompt = 'Your task is to extract the content of the page. You will be given a page and a goal and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. Extraction goal: {goal}, Page: {page}'
|
||||
prompt_with_outputlimit = 'Your task is to extract the content of the page. You will be given a page and a goal and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. Extraction goal: {goal}, Page: {page} \n\n#The length of the returned result must be less than {max_extract_content_output_tokens} characters.'
|
||||
template = PromptTemplate(input_variables=['goal', 'page'], template=prompt)
|
||||
|
||||
messages = [{'role': 'user', 'content': template.format(goal=goal, page=content)}]
|
||||
try:
|
||||
output = call_llm_model(llm,
|
||||
messages=messages,
|
||||
model=llm_config.llm_model_name,
|
||||
temperature=llm_config.llm_temperature)
|
||||
result_content = output.content
|
||||
|
||||
# Check if output exceeds the token limit and retry with length-limited prompt if needed
|
||||
if max_extract_content_output_tokens and len(result_content) > max_extract_content_output_tokens:
|
||||
logger.info(
|
||||
f"Output exceeds maximum length ({len(result_content)} > {max_extract_content_output_tokens}). Retrying with limited prompt.")
|
||||
template_with_limit = PromptTemplate(
|
||||
input_variables=['goal', 'page', 'max_extract_content_output_tokens'],
|
||||
template=prompt_with_outputlimit
|
||||
)
|
||||
messages = [{'role': 'user', 'content': template_with_limit.format(
|
||||
goal=goal,
|
||||
page=content,
|
||||
max_extract_content_output_tokens=max_extract_content_output_tokens,
|
||||
max_tokens=max_extract_content_output_tokens
|
||||
)}]
|
||||
# extract content with length limit
|
||||
output = call_llm_model(llm,
|
||||
messages=messages,
|
||||
model=llm_config.llm_model_name,
|
||||
temperature=llm_config.llm_temperature)
|
||||
result_content = output.content
|
||||
|
||||
msg = f'Extracted from page\n: {result_content}\n'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
except Exception as e:
|
||||
logger.debug(f'Error extracting content: {e}')
|
||||
msg = f'Extracted from page\n: {content}\n'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg), page
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.SCROLL_DOWN.value.name,
|
||||
desc=BrowserAction.SCROLL_DOWN.value.desc,
|
||||
tool_name="browser")
|
||||
class ScrollDown(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SCROLL_DOWN.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.SCROLL_DOWN.name} page is none")
|
||||
return ActionResult(content="scroll no page", keep=True), page
|
||||
|
||||
amount = action.params.get("amount")
|
||||
if not amount:
|
||||
page.evaluate('window.scrollBy(0, window.innerHeight);')
|
||||
else:
|
||||
amount = int(amount)
|
||||
page.evaluate(f'window.scrollBy(0, {amount});')
|
||||
|
||||
amount = f'{amount} pixels' if amount else 'one page'
|
||||
msg = f'Scrolled down the page by {amount}'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SCROLL_DOWN.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.SCROLL_DOWN.name} page is none")
|
||||
return ActionResult(content="scroll no page", keep=True), page
|
||||
|
||||
amount = action.params.get("amount")
|
||||
if not amount:
|
||||
await page.evaluate('window.scrollBy(0, window.innerHeight);')
|
||||
else:
|
||||
amount = int(amount)
|
||||
await page.evaluate(f'window.scrollBy(0, {amount});')
|
||||
|
||||
amount = f'{amount} pixels' if amount else 'one page'
|
||||
msg = f'Scrolled down the page by {amount}'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.SCROLL_UP.value.name,
|
||||
desc=BrowserAction.SCROLL_UP.value.desc,
|
||||
tool_name="browser")
|
||||
class ScrollUp(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SCROLL_UP.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.SCROLL_UP.name} page is none")
|
||||
return ActionResult(content="scroll no page", keep=True), page
|
||||
|
||||
amount = action.params.get("amount")
|
||||
if not amount:
|
||||
page.evaluate('window.scrollBy(0, -window.innerHeight);')
|
||||
else:
|
||||
amount = int(amount)
|
||||
page.evaluate(f'window.scrollBy(0, -{amount});')
|
||||
|
||||
amount = f'{amount} pixels' if amount else 'one page'
|
||||
msg = f'Scrolled down the page by {amount}'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SCROLL_UP.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.SCROLL_UP.name} page is none")
|
||||
return ActionResult(content="scroll no page", keep=True), page
|
||||
|
||||
amount = action.params.get("amount")
|
||||
if not amount:
|
||||
await page.evaluate('window.scrollBy(0, -window.innerHeight);')
|
||||
else:
|
||||
amount = int(amount)
|
||||
await page.evaluate(f'window.scrollBy(0, -{amount});')
|
||||
|
||||
amount = f'{amount} pixels' if amount else 'one page'
|
||||
msg = f'Scrolled down the page by {amount}'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.WAIT.value.name,
|
||||
desc=BrowserAction.WAIT.value.desc,
|
||||
tool_name="browser")
|
||||
class Wait(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
seconds = action.params.get("seconds")
|
||||
if not seconds:
|
||||
seconds = action.params.get("duration", 0)
|
||||
seconds = int(seconds)
|
||||
msg = f'Waiting for {seconds} seconds'
|
||||
logger.info(msg)
|
||||
time.sleep(seconds)
|
||||
return ActionResult(content=msg, keep=True), kwargs.get('page')
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
seconds = action.params.get("seconds")
|
||||
if not seconds:
|
||||
seconds = action.params.get("duration", 0)
|
||||
seconds = int(seconds)
|
||||
msg = f'Waiting for {seconds} seconds'
|
||||
logger.info(msg)
|
||||
await asyncio.sleep(seconds)
|
||||
return ActionResult(content=msg, keep=True), kwargs.get('page')
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.SWITCH_TAB.value.name,
|
||||
desc=BrowserAction.SWITCH_TAB.value.desc,
|
||||
tool_name="browser")
|
||||
class SwitchTab(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SWITCH_TAB.value.name} action")
|
||||
browser = get_browser(**kwargs)
|
||||
if browser is None:
|
||||
logger.warning(f"{BrowserAction.SWITCH_TAB.name} browser context is none")
|
||||
return ActionResult(content="switch tab no browser context", keep=True), get_page(**kwargs)
|
||||
|
||||
page_id = action.params.get("page_id", 0)
|
||||
page_id = int(page_id)
|
||||
pages = browser.pages
|
||||
|
||||
if page_id >= len(pages):
|
||||
raise RuntimeError(f'No tab found with page_id: {page_id}')
|
||||
|
||||
page = pages[page_id]
|
||||
page.bring_to_front()
|
||||
page.wait_for_load_state()
|
||||
msg = f'Switched to tab {page_id}'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SWITCH_TAB.value.name} action")
|
||||
browser = get_browser(**kwargs)
|
||||
if browser is None:
|
||||
logger.warning(f"{BrowserAction.SWITCH_TAB.name} browser context is none")
|
||||
return ActionResult(content="switch tab no browser context", keep=True), get_page(**kwargs)
|
||||
|
||||
page_id = action.params.get("page_id", 0)
|
||||
page_id = int(page_id)
|
||||
pages = browser.pages
|
||||
|
||||
if page_id >= len(pages):
|
||||
raise RuntimeError(f'No tab found with page_id: {page_id}')
|
||||
|
||||
page = pages[page_id]
|
||||
await page.bring_to_front()
|
||||
await page.wait_for_load_state()
|
||||
msg = f'Switched to tab {page_id}'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), page
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.SEND_KEYS.value.name,
|
||||
desc=BrowserAction.SEND_KEYS.value.desc,
|
||||
tool_name="browser")
|
||||
class SendKeys(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SEND_KEYS.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.SEND_KEYS.name} page is none")
|
||||
return ActionResult(content="scroll no page", keep=True), page
|
||||
|
||||
keys = action.params.get("keys")
|
||||
if not keys:
|
||||
return ActionResult(success=False, content="no keys", keep=True), page
|
||||
|
||||
try:
|
||||
page.keyboard.press(keys)
|
||||
except Exception as e:
|
||||
logger.warning(f"{keys} press fail. \n{traceback.format_exc()}")
|
||||
raise e
|
||||
return ActionResult(content=f"Sent keys: {keys}", keep=True), page
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.SEND_KEYS.value.name} action")
|
||||
page = get_page(**kwargs)
|
||||
if page is None:
|
||||
logger.warning(f"{BrowserAction.SEND_KEYS.name} page is none")
|
||||
return ActionResult(content="scroll no page", keep=True), page
|
||||
|
||||
keys = action.params.get("keys")
|
||||
if not keys:
|
||||
return ActionResult(success=False, content="no keys", keep=True), page
|
||||
|
||||
try:
|
||||
await page.keyboard.press(keys)
|
||||
except Exception as e:
|
||||
logger.warning(f"{keys} press fail. \n{traceback.format_exc()}")
|
||||
raise e
|
||||
|
||||
return ActionResult(content=f"Sent keys: {keys}", keep=True), page
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.WRITE_TO_FILE.value.name,
|
||||
desc=BrowserAction.WRITE_TO_FILE.value.desc,
|
||||
tool_name="browser")
|
||||
class WriteToFile(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
# 设置默认文件路径
|
||||
file_path = "tmp_result.md"
|
||||
# 检查参数中是否有file_path
|
||||
if "file_path" in action.params:
|
||||
file_path = action.params.get("file_path", "tmp_result.md")
|
||||
# 检查参数中是否有file_name
|
||||
elif "file_name" in action.params:
|
||||
file_path = action.params.get("file_name", "tmp_result.md")
|
||||
elif "filename" in action.params:
|
||||
file_path = action.params.get("filename", "tmp_result.md")
|
||||
content = action.params.get("content", "")
|
||||
mode = action.params.get("mode", "a") # Default to append mode
|
||||
# 获取文件的绝对路径
|
||||
abs_file_path = os.path.abspath(file_path)
|
||||
try:
|
||||
with open(file_path, mode, encoding='utf-8') as f:
|
||||
f.write(content + '\n')
|
||||
msg = f'Successfully wrote content to {abs_file_path}'
|
||||
logger.info(msg)
|
||||
return ActionResult(content=msg, keep=True), get_page(**kwargs)
|
||||
except Exception as e:
|
||||
error_msg = f'Failed to write to file {abs_file_path}: {str(e)}'
|
||||
logger.error(error_msg)
|
||||
return ActionResult(content=error_msg, keep=True, error=error_msg), get_page(**kwargs)
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
# For file operations, we don't need to make this asynchronous
|
||||
return self.act(action, **kwargs)
|
||||
|
||||
|
||||
@ActionFactory.register(name=BrowserAction.DONE.value.name,
|
||||
desc=BrowserAction.DONE.value.desc,
|
||||
tool_name="browser")
|
||||
class Done(ExecutableAction):
|
||||
def act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.DONE.value.name} action")
|
||||
return ActionResult(is_done=True, success=True, content="done", keep=True), get_page(**kwargs)
|
||||
|
||||
async def async_act(self, action: ActionModel, **kwargs) -> Tuple[ActionResult, Any]:
|
||||
logger.info(f"exec {BrowserAction.DONE.value.name} action")
|
||||
return ActionResult(is_done=True, success=True, content="done", keep=True), get_page(**kwargs)
|
||||
@@ -0,0 +1,71 @@
|
||||
# coding: utf-8
|
||||
# Copyright (c) 2025 inclusionAI.
|
||||
|
||||
from typing import Tuple, List, Any
|
||||
|
||||
from aworld.core.tool.action_factory import ActionFactory
|
||||
from aworld.core.common import ActionModel, ActionResult, Observation
|
||||
from aworld.logs.util import logger
|
||||
from aworld.core.tool.base import Tool, ToolActionExecutor
|
||||
|
||||
|
||||
class BrowserToolActionExecutor(ToolActionExecutor):
|
||||
def __init__(self, tool: Tool = None):
|
||||
super(BrowserToolActionExecutor, self).__init__(tool)
|
||||
|
||||
def execute_action(self, actions: List[ActionModel], **kwargs) -> Tuple[
|
||||
List[ActionResult], Any]:
|
||||
"""Execute the specified browser action sequence by agent policy.
|
||||
|
||||
Args:
|
||||
actions: Tool action sequence.
|
||||
|
||||
Returns:
|
||||
Browser page and action result list.
|
||||
"""
|
||||
action_results = []
|
||||
page = self.tool.page
|
||||
for action in actions:
|
||||
action_result, page = self._exec(action, **kwargs)
|
||||
action_results.append(action_result)
|
||||
return action_results, page
|
||||
|
||||
async def async_execute_action(self, actions: List[ActionModel], **kwargs) -> Tuple[
|
||||
List[ActionResult], Any]:
|
||||
"""Execute the specified browser action sequence by agent policy.
|
||||
|
||||
Args:
|
||||
actions: Tool action sequence.
|
||||
|
||||
Returns:
|
||||
Browser page and action result list.
|
||||
"""
|
||||
action_results = []
|
||||
page = self.tool.page
|
||||
for action in actions:
|
||||
action_result, page = await self._async_exec(action, **kwargs)
|
||||
action_results.append(action_result)
|
||||
return action_results, page
|
||||
|
||||
def _exec(self, action_model: ActionModel, **kwargs):
|
||||
action_name = action_model.action_name
|
||||
if action_name not in ActionFactory:
|
||||
raise ValueError(f'Action {action_name} not found')
|
||||
|
||||
action = ActionFactory(action_name)
|
||||
action_result, page = action.act(action_model, page=self.tool.page, browser=self.tool.browser_context, **kwargs)
|
||||
logger.info(f"{action_name} execute finished")
|
||||
return action_result, page
|
||||
|
||||
async def _async_exec(self, action_model: ActionModel, **kwargs):
|
||||
action_name = action_model.action_name
|
||||
if action_name not in ActionFactory:
|
||||
action_name = action_model.tool_name + action_model.action_name
|
||||
if action_name not in ActionFactory:
|
||||
raise ValueError(f'Action {action_name} not found')
|
||||
|
||||
action = ActionFactory(action_name)
|
||||
action_result, page = await action.async_act(action_model, page=self.tool.page,
|
||||
browser=self.tool.browser_context, **kwargs)
|
||||
logger.info(f"{action_name} execute finished")
|
||||
return action_result, page
|
||||
@@ -0,0 +1,507 @@
|
||||
# coding: utf-8
|
||||
# Copyright (c) 2025 inclusionAI.
|
||||
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
from typing import Optional
|
||||
|
||||
from examples.common.tools.browsers.util.dom import DOMElementNode
|
||||
from aworld.logs.util import logger
|
||||
from aworld.utils import import_package
|
||||
|
||||
|
||||
class DomUtil:
|
||||
def __init__(self):
|
||||
import_package("playwright")
|
||||
|
||||
@staticmethod
|
||||
async def async_click_element(page, element_node: DOMElementNode, **kwargs) -> Optional[str]:
|
||||
from playwright.async_api import ElementHandle as AElementHandle, BrowserContext as ABrowserContext
|
||||
|
||||
try:
|
||||
element_handle: AElementHandle = await DomUtil.async_get_locate_element(page, element_node)
|
||||
if element_handle is None:
|
||||
raise Exception(f'Element: {repr(element_node)} not found')
|
||||
|
||||
bound = await element_handle.bounding_box()
|
||||
try:
|
||||
# todo: iframe.
|
||||
center_x = bound['x'] + bound['width'] / 2
|
||||
center_y = bound['y'] + bound['height'] / 2
|
||||
|
||||
try:
|
||||
browser: ABrowserContext = kwargs.get('browser')
|
||||
async with browser.expect_page() as new_page_info:
|
||||
await page.mouse.click(center_x, center_y)
|
||||
await page.mouse.click(center_x, center_y)
|
||||
await page.wait_for_load_state()
|
||||
except:
|
||||
logger.warning(traceback.format_exc())
|
||||
except:
|
||||
logger.info(f"click {element_handle}!!")
|
||||
if await element_handle.text_content():
|
||||
browser: ABrowserContext = kwargs.get('browser')
|
||||
if browser:
|
||||
try:
|
||||
async with browser.expect_page() as new_page_info:
|
||||
await page.click(f"text={element_handle.text_content()}")
|
||||
page = await new_page_info.value
|
||||
await page.wait_for_load_state()
|
||||
except:
|
||||
logger.warning(traceback.format_exc())
|
||||
else:
|
||||
await element_handle.click()
|
||||
await page.wait_for_load_state()
|
||||
else:
|
||||
await element_handle.click()
|
||||
await page.wait_for_load_state()
|
||||
except Exception as e:
|
||||
logger.error(traceback.format_exc())
|
||||
raise Exception(f'Failed to click element: {repr(element_node)}. Error: {str(e)}')
|
||||
|
||||
@staticmethod
|
||||
def click_element(page, element_node: DOMElementNode, **kwargs) -> Optional[str]:
|
||||
from playwright.sync_api import ElementHandle, BrowserContext
|
||||
|
||||
try:
|
||||
element_handle: ElementHandle = DomUtil.get_locate_element(page, element_node)
|
||||
if element_handle is None:
|
||||
raise Exception(f'Element: {repr(element_node)} not found')
|
||||
|
||||
bound = element_handle.bounding_box()
|
||||
try:
|
||||
# todo: iframe.
|
||||
center_x = bound['x'] + bound['width'] / 2
|
||||
center_y = bound['y'] + bound['height'] / 2
|
||||
|
||||
try:
|
||||
browser: BrowserContext = kwargs.get('browser')
|
||||
with browser.expect_page() as new_page_info:
|
||||
page.mouse.click(center_x, center_y)
|
||||
page = new_page_info.value
|
||||
page.wait_for_load_state()
|
||||
except:
|
||||
logger.warning(traceback.format_exc())
|
||||
except:
|
||||
logger.info(f"click {element_handle}!!")
|
||||
if element_handle.text_content():
|
||||
browser: BrowserContext = kwargs.get('browser')
|
||||
if browser:
|
||||
try:
|
||||
with browser.expect_page() as new_page_info:
|
||||
page.click(f"text={element_handle.text_content()}")
|
||||
page = new_page_info.value
|
||||
page.wait_for_load_state()
|
||||
except:
|
||||
logger.warning(traceback.format_exc())
|
||||
else:
|
||||
element_handle.click()
|
||||
page.wait_for_load_state()
|
||||
else:
|
||||
element_handle.click()
|
||||
page.wait_for_load_state()
|
||||
except Exception as e:
|
||||
logger.error(traceback.format_exc())
|
||||
raise Exception(f'Failed to click element: {repr(element_node)}. Error: {str(e)}')
|
||||
|
||||
@staticmethod
|
||||
async def async_get_locate_element(current_frame, element: DOMElementNode):
|
||||
# Start with the target element and collect all parents, return Optional[AElementHandle]
|
||||
from playwright.async_api import FrameLocator as AFrameLocator
|
||||
|
||||
parents: list[DOMElementNode] = []
|
||||
current = element
|
||||
while current.parent is not None:
|
||||
parent = current.parent
|
||||
parents.append(parent)
|
||||
current = parent
|
||||
|
||||
# Reverse the parents list to process from top to bottom
|
||||
parents.reverse()
|
||||
|
||||
# Process all iframe parents in sequence
|
||||
iframes = [item for item in parents if item.tag_name == 'iframe']
|
||||
for parent in iframes:
|
||||
css_selector = DomUtil._enhanced_css_selector_for_element(
|
||||
parent,
|
||||
include_dynamic_attributes=True,
|
||||
)
|
||||
current_frame = current_frame.frame_locator(css_selector)
|
||||
|
||||
css_selector = DomUtil._enhanced_css_selector_for_element(
|
||||
element, include_dynamic_attributes=True
|
||||
)
|
||||
|
||||
try:
|
||||
if isinstance(current_frame, AFrameLocator):
|
||||
element_handle = await current_frame.locator(css_selector).element_handle()
|
||||
return element_handle
|
||||
else:
|
||||
# Try to scroll into view if hidden
|
||||
element_handle = await current_frame.query_selector(css_selector)
|
||||
if element_handle:
|
||||
await element_handle.scroll_into_view_if_needed()
|
||||
return element_handle
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to locate element: {str(e)}')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_locate_element(current_frame, element: DOMElementNode):
|
||||
# Start with the target element and collect all parents
|
||||
from playwright.sync_api import FrameLocator
|
||||
|
||||
parents: list[DOMElementNode] = []
|
||||
current = element
|
||||
while current.parent is not None:
|
||||
parent = current.parent
|
||||
parents.append(parent)
|
||||
current = parent
|
||||
|
||||
# Reverse the parents list to process from top to bottom
|
||||
parents.reverse()
|
||||
|
||||
# Process all iframe parents in sequence
|
||||
iframes = [item for item in parents if item.tag_name == 'iframe']
|
||||
for parent in iframes:
|
||||
css_selector = DomUtil._enhanced_css_selector_for_element(
|
||||
parent,
|
||||
include_dynamic_attributes=True,
|
||||
)
|
||||
current_frame = current_frame.frame_locator(css_selector)
|
||||
|
||||
css_selector = DomUtil._enhanced_css_selector_for_element(
|
||||
element, include_dynamic_attributes=True
|
||||
)
|
||||
|
||||
try:
|
||||
if isinstance(current_frame, FrameLocator):
|
||||
element_handle = current_frame.locator(css_selector).element_handle()
|
||||
return element_handle
|
||||
else:
|
||||
# Try to scroll into view if hidden
|
||||
element_handle = current_frame.query_selector(css_selector)
|
||||
if element_handle:
|
||||
element_handle.scroll_into_view_if_needed()
|
||||
return element_handle
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to locate element: {str(e)}')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def wait_for_stable_network(page, **kwargs):
|
||||
pending_requests = set()
|
||||
last_activity = time.time()
|
||||
|
||||
# Define relevant resource types and content types
|
||||
RELEVANT_RESOURCE_TYPES = {
|
||||
'document',
|
||||
'stylesheet',
|
||||
'image',
|
||||
'font',
|
||||
'script',
|
||||
'iframe',
|
||||
}
|
||||
|
||||
RELEVANT_CONTENT_TYPES = {
|
||||
'text/html',
|
||||
'text/css',
|
||||
'application/javascript',
|
||||
'image/',
|
||||
'font/',
|
||||
'application/json',
|
||||
}
|
||||
|
||||
# Additional patterns to filter out
|
||||
IGNORED_URL_PATTERNS = {
|
||||
# Analytics and tracking
|
||||
'analytics',
|
||||
'tracking',
|
||||
'telemetry',
|
||||
'beacon',
|
||||
'metrics',
|
||||
# Ad-related
|
||||
'doubleclick',
|
||||
'adsystem',
|
||||
'adserver',
|
||||
'advertising',
|
||||
# Social media widgets
|
||||
'facebook.com/plugins',
|
||||
'platform.twitter',
|
||||
'linkedin.com/embed',
|
||||
# Live chat and support
|
||||
'livechat',
|
||||
'zendesk',
|
||||
'intercom',
|
||||
'crisp.chat',
|
||||
'hotjar',
|
||||
# Push notifications
|
||||
'push-notifications',
|
||||
'onesignal',
|
||||
'pushwoosh',
|
||||
# Background sync/heartbeat
|
||||
'heartbeat',
|
||||
'ping',
|
||||
'alive',
|
||||
# WebRTC and streaming
|
||||
'webrtc',
|
||||
'rtmp://',
|
||||
'wss://',
|
||||
# Common CDNs for dynamic content
|
||||
'cloudfront.net',
|
||||
'fastly.net',
|
||||
}
|
||||
|
||||
def on_request(request):
|
||||
# Filter by resource type
|
||||
if request.resource_type not in RELEVANT_RESOURCE_TYPES:
|
||||
return
|
||||
|
||||
# Filter out streaming, websocket, and other real-time requests
|
||||
if request.resource_type in {
|
||||
'websocket',
|
||||
'media',
|
||||
'eventsource',
|
||||
'manifest',
|
||||
'other',
|
||||
}:
|
||||
return
|
||||
|
||||
# Filter out by URL patterns
|
||||
url = request.url.lower()
|
||||
if any(pattern in url for pattern in IGNORED_URL_PATTERNS):
|
||||
return
|
||||
|
||||
# Filter out data URLs and blob URLs
|
||||
if url.startswith(('data:', 'blob:')):
|
||||
return
|
||||
|
||||
# Filter out requests with certain headers
|
||||
headers = request.headers
|
||||
if headers.get('purpose') == 'prefetch' or headers.get('sec-fetch-dest') in [
|
||||
'video',
|
||||
'audio',
|
||||
]:
|
||||
return
|
||||
|
||||
nonlocal last_activity
|
||||
pending_requests.add(request)
|
||||
last_activity = time.time()
|
||||
|
||||
def on_response(response):
|
||||
request = response.request
|
||||
if request not in pending_requests:
|
||||
return
|
||||
|
||||
# Filter by content type if available
|
||||
content_type = response.headers.get('content-type', '').lower()
|
||||
|
||||
# Skip if content type indicates streaming or real-time data
|
||||
if any(t in content_type
|
||||
for t in [
|
||||
'streaming',
|
||||
'video',
|
||||
'audio',
|
||||
'webm',
|
||||
'mp4',
|
||||
'event-stream',
|
||||
'websocket',
|
||||
'protobuf']):
|
||||
pending_requests.remove(request)
|
||||
return
|
||||
|
||||
# Only process relevant content types
|
||||
if not any(ct in content_type for ct in RELEVANT_CONTENT_TYPES):
|
||||
pending_requests.remove(request)
|
||||
return
|
||||
|
||||
# Skip if response is too large (likely not essential for page load)
|
||||
content_length = response.headers.get('content-length')
|
||||
if content_length and int(content_length) > 5 * 1024 * 1024: # 5MB
|
||||
pending_requests.remove(request)
|
||||
return
|
||||
|
||||
nonlocal last_activity
|
||||
pending_requests.remove(request)
|
||||
last_activity = time.time()
|
||||
|
||||
# Attach event listeners
|
||||
page.on('request', on_request)
|
||||
page.on('response', on_response)
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
while True:
|
||||
time.sleep(0.1)
|
||||
now = time.time()
|
||||
|
||||
if len(pending_requests) == 0 and (now - last_activity) >= kwargs.get('idle_wait_time', 0.5):
|
||||
break
|
||||
if now - start_time > kwargs.get('max_wait_time', 5):
|
||||
logger.debug(
|
||||
f'Network timeout after {kwargs.get("max_wait_time", 5)}s with {len(pending_requests)} '
|
||||
f'pending requests: {[r.url for r in pending_requests]}'
|
||||
)
|
||||
break
|
||||
|
||||
finally:
|
||||
# Clean up event listeners
|
||||
page.remove_listener('request', on_request)
|
||||
page.remove_listener('response', on_response)
|
||||
logger.debug(f'Network stabilized for {kwargs.get("idle_wait_time", 0.5)} seconds')
|
||||
|
||||
@staticmethod
|
||||
def _enhanced_css_selector_for_element(element: DOMElementNode, include_dynamic_attributes: bool = True) -> str:
|
||||
"""Creates a CSS selector for a DOM element, handling various edge cases and special characters.
|
||||
|
||||
Args:
|
||||
element: The DOM element to create a selector for
|
||||
|
||||
Returns:
|
||||
A valid CSS selector string
|
||||
"""
|
||||
try:
|
||||
# Get base selector from XPath
|
||||
css_selector = DomUtil._convert_simple_xpath_to_css_selector(element.xpath)
|
||||
|
||||
# Handle class attributes
|
||||
if 'class' in element.attributes and element.attributes['class'] and include_dynamic_attributes:
|
||||
# Define a regex pattern for valid class names in CSS
|
||||
valid_class_name_pattern = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_-]*$')
|
||||
|
||||
# Iterate through the class attribute values
|
||||
classes = element.attributes['class'].split()
|
||||
for class_name in classes:
|
||||
# Skip empty class names
|
||||
if not class_name.strip():
|
||||
continue
|
||||
|
||||
# Check if the class name is valid
|
||||
if valid_class_name_pattern.match(class_name):
|
||||
# Append the valid class name to the CSS selector
|
||||
css_selector += f'.{class_name}'
|
||||
else:
|
||||
# Skip invalid class names
|
||||
continue
|
||||
|
||||
# Expanded set of safe attributes that are stable and useful for selection
|
||||
SAFE_ATTRIBUTES = {
|
||||
# Data attributes (if they're stable in your application)
|
||||
'id',
|
||||
# Standard HTML attributes
|
||||
'name',
|
||||
'type',
|
||||
'placeholder',
|
||||
# Accessibility attributes
|
||||
'aria-label',
|
||||
'aria-labelledby',
|
||||
'aria-describedby',
|
||||
'role',
|
||||
# Common form attributes
|
||||
'for',
|
||||
'autocomplete',
|
||||
'required',
|
||||
'readonly',
|
||||
# Media attributes
|
||||
'alt',
|
||||
'title',
|
||||
'src',
|
||||
# Custom stable attributes (add any application-specific ones)
|
||||
'href',
|
||||
'target',
|
||||
}
|
||||
|
||||
if include_dynamic_attributes:
|
||||
dynamic_attributes = {
|
||||
'data-id',
|
||||
'data-qa',
|
||||
'data-cy',
|
||||
'data-testid',
|
||||
}
|
||||
SAFE_ATTRIBUTES.update(dynamic_attributes)
|
||||
|
||||
# Handle other attributes
|
||||
for attribute, value in element.attributes.items():
|
||||
if attribute == 'class':
|
||||
continue
|
||||
|
||||
# Skip invalid attribute names
|
||||
if not attribute.strip():
|
||||
continue
|
||||
|
||||
if attribute not in SAFE_ATTRIBUTES:
|
||||
continue
|
||||
|
||||
# Escape special characters in attribute names
|
||||
safe_attribute = attribute.replace(':', r'\:')
|
||||
|
||||
# Handle different value cases
|
||||
if value == '':
|
||||
css_selector += f'[{safe_attribute}]'
|
||||
elif any(char in value for char in '"\'<>`\n\r\t'):
|
||||
# Use contains for values with special characters
|
||||
# Regex-substitute *any* whitespace with a single space, then strip.
|
||||
collapsed_value = re.sub(r'\s+', ' ', value).strip()
|
||||
# Escape embedded double-quotes.
|
||||
safe_value = collapsed_value.replace('"', '\\"')
|
||||
css_selector += f'[{safe_attribute}*="{safe_value}"]'
|
||||
else:
|
||||
css_selector += f'[{safe_attribute}="{value}"]'
|
||||
|
||||
return css_selector
|
||||
|
||||
except Exception:
|
||||
# Fallback to a more basic selector if something goes wrong
|
||||
tag_name = element.tag_name or '*'
|
||||
return f"{tag_name}[highlight_index='{element.highlight_index}']"
|
||||
|
||||
@staticmethod
|
||||
def _convert_simple_xpath_to_css_selector(xpath: str) -> str:
|
||||
"""Converts simple XPath expressions to CSS selectors."""
|
||||
if not xpath:
|
||||
return ''
|
||||
|
||||
# Remove leading slash if present
|
||||
xpath = xpath.lstrip('/')
|
||||
|
||||
# Split into parts
|
||||
parts = xpath.split('/')
|
||||
css_parts = []
|
||||
|
||||
for part in parts:
|
||||
if not part:
|
||||
continue
|
||||
|
||||
# Handle index notation [n]
|
||||
if '[' in part:
|
||||
base_part = part[: part.find('[')]
|
||||
index_part = part[part.find('['):]
|
||||
|
||||
# Handle multiple indices
|
||||
indices = [i.strip('[]') for i in index_part.split(']')[:-1]]
|
||||
|
||||
for idx in indices:
|
||||
try:
|
||||
# Handle numeric indices
|
||||
if idx.isdigit():
|
||||
index = int(idx) - 1
|
||||
base_part += f':nth-of-type({index + 1})'
|
||||
# Handle last() function
|
||||
elif idx == 'last()':
|
||||
base_part += ':last-of-type'
|
||||
# Handle position() functions
|
||||
elif 'position()' in idx:
|
||||
if '>1' in idx:
|
||||
base_part += ':nth-of-type(n+2)'
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
css_parts.append(base_part)
|
||||
else:
|
||||
css_parts.append(part)
|
||||
|
||||
base_selector = ' > '.join(css_parts)
|
||||
return base_selector
|
||||
@@ -0,0 +1,361 @@
|
||||
# coding: utf-8
|
||||
# Copyright (c) 2025 inclusionAI.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import traceback
|
||||
from importlib import resources
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Tuple, List
|
||||
|
||||
from examples.common.tools.common import package
|
||||
from examples.common.tools.tool_action import BrowserAction
|
||||
from aworld.core.common import Observation, ActionModel, ActionResult
|
||||
from aworld.logs.util import logger
|
||||
from aworld.core.tool.base import action_executor, ToolFactory, AsyncTool
|
||||
from aworld.utils.import_package import is_package_installed
|
||||
from examples.common.tools.browsers.action.executor import BrowserToolActionExecutor
|
||||
from examples.common.tools.browsers.util.dom import DomTree
|
||||
from examples.common.tools.conf import BrowserToolConfig
|
||||
from examples.common.tools.browsers.util.dom_build import async_build_dom_tree
|
||||
from aworld.utils import import_package
|
||||
from aworld.tools.utils import build_observation
|
||||
|
||||
URL_MAX_LENGTH = 4096
|
||||
UTF8 = "".join(chr(x) for x in range(0, 55290))
|
||||
ASCII = "".join(chr(x) for x in range(32, 128))
|
||||
|
||||
|
||||
@ToolFactory.register(name="browser",
|
||||
desc="browser",
|
||||
asyn=True,
|
||||
supported_action=BrowserAction,
|
||||
conf_file_name=f'browser_tool.yaml',
|
||||
dir=f"{Path(__file__).parent.absolute()}")
|
||||
class BrowserTool(AsyncTool):
|
||||
def __init__(self, conf: BrowserToolConfig, **kwargs) -> None:
|
||||
super(BrowserTool, self).__init__(conf)
|
||||
|
||||
self.initialized = False
|
||||
self._finish = False
|
||||
self.record_trace = self.conf.get("working_dir", False)
|
||||
self.sleep_after_init = self.conf.get("sleep_after_init", False)
|
||||
dom_js_path = self.conf.get('dom_js_path')
|
||||
if dom_js_path and os.path.exists(dom_js_path):
|
||||
with open(dom_js_path, 'r') as read:
|
||||
self.js_code = read.read()
|
||||
else:
|
||||
self.js_code = resources.read_text(f'{package}.browsers.script',
|
||||
'buildDomTree.js')
|
||||
self.cur_observation = None
|
||||
if not is_package_installed('playwright'):
|
||||
import_package("playwright")
|
||||
logger.info("playwright install...")
|
||||
try:
|
||||
subprocess.check_call('playwright install', shell=True, timeout=300)
|
||||
except Exception as e:
|
||||
logger.error(f"Fail to auto execute playwright install, you can install manually\n {e}")
|
||||
|
||||
async def init(self) -> None:
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
if self.initialized:
|
||||
return
|
||||
|
||||
self.context_manager = async_playwright()
|
||||
self.playwright = await self.context_manager.start()
|
||||
|
||||
self.browser = await self._create_browser()
|
||||
self.browser_context = await self._create_browser_context()
|
||||
|
||||
if self.record_trace:
|
||||
await self.browser_context.tracing.start(screenshots=True, snapshots=True)
|
||||
|
||||
self.page = await self.browser_context.new_page()
|
||||
if self.conf.get("custom_executor"):
|
||||
self.action_executor = BrowserToolActionExecutor(self)
|
||||
else:
|
||||
self.action_executor = action_executor
|
||||
self.initialized = True
|
||||
|
||||
async def _create_browser(self):
|
||||
browse_name = self.conf.get("browse_name", "chromium")
|
||||
browse = getattr(self.playwright, browse_name)
|
||||
cdp_url = self.conf.get("cdp_url")
|
||||
wss_url = self.conf.get("wss_url")
|
||||
if cdp_url:
|
||||
if browse_name != "chromium":
|
||||
logger.warning(f"{browse_name} unsupported CDP, will use chromium browser")
|
||||
browse = self.playwright.chromium
|
||||
logger.info(f"Connecting to remote browser via CDP {cdp_url}")
|
||||
browser = await browse.connect_over_cdp(cdp_url)
|
||||
elif wss_url:
|
||||
logger.info(f"Connecting to remote browser via wss {wss_url}")
|
||||
browser = await browse.connect(wss_url)
|
||||
else:
|
||||
headless = self.conf.get("headless", False)
|
||||
slow_mo = self.conf.get("slow_mo", 0)
|
||||
disable_security_args = []
|
||||
if self.conf.get('disable_security', False):
|
||||
disable_security_args = ['--disable-web-security',
|
||||
'--disable-site-isolation-trials',
|
||||
'--disable-features=IsolateOrigins,site-per-process']
|
||||
args = ['--no-sandbox',
|
||||
'--disable-crash-reporte',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-infobars',
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-popup-blocking',
|
||||
'--disable-backgrounding-occluded-windows',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--disable-window-activation',
|
||||
'--disable-focus-on-load',
|
||||
'--no-first-run',
|
||||
'--no-default-browser-check',
|
||||
'--no-startup-window',
|
||||
'--window-position=0,0',
|
||||
'--window-size=1280,720'] + disable_security_args
|
||||
browser = await browse.launch(
|
||||
headless=headless,
|
||||
slow_mo=slow_mo,
|
||||
args=args,
|
||||
proxy=self.conf.get('proxy'),
|
||||
)
|
||||
return browser
|
||||
|
||||
async def _create_browser_context(self):
|
||||
"""Creates a new browser context with anti-detection measures and loads cookies if available."""
|
||||
from playwright.async_api import ViewportSize
|
||||
|
||||
browser = self.browser
|
||||
if self.conf.get("cdp_url") and len(browser.contexts) > 0:
|
||||
context = browser.contexts[0]
|
||||
else:
|
||||
viewport_size = ViewportSize(width=self.conf.get("width", 1280),
|
||||
height=self.conf.get("height", 720))
|
||||
disable_security = self.conf.get('disable_security', False)
|
||||
|
||||
context = await browser.new_context(viewport=viewport_size,
|
||||
no_viewport=False,
|
||||
user_agent=self.conf.get('user_agent'),
|
||||
java_script_enabled=True,
|
||||
bypass_csp=disable_security,
|
||||
ignore_https_errors=disable_security,
|
||||
record_video_dir=self.conf.get('working_dir'),
|
||||
record_video_size=viewport_size,
|
||||
locale=self.conf.get('locale'),
|
||||
storage_state=self.conf.get("storage_state", None),
|
||||
geolocation=self.conf.get("geolocation", None),
|
||||
device_scale_factor=1)
|
||||
if "chromium" == self.conf.get("browse_name", "chromium"):
|
||||
await context.grant_permissions(['camera', 'microphone'])
|
||||
|
||||
if self.conf.get('trace_path'):
|
||||
await context.tracing.start(screenshots=True, snapshots=True, sources=True)
|
||||
|
||||
cookie_file = self.conf.get('cookies_file')
|
||||
if cookie_file and os.path.exists(cookie_file):
|
||||
with open(cookie_file, 'r') as read:
|
||||
cookies = json.loads(read.read())
|
||||
await context.add_cookies(cookies)
|
||||
logger.info(f'Cookies load from {cookie_file} finished')
|
||||
|
||||
if self.conf.get('private'):
|
||||
js = resources.read_text(f"{package}.browsers.script", "stealth.min.js")
|
||||
await context.add_init_script(js)
|
||||
|
||||
return context
|
||||
|
||||
async def get_cur_page(self):
|
||||
return self.page
|
||||
|
||||
async def screenshot(self, full_page: bool = False) -> str:
|
||||
"""Returns a base64 encoded screenshot of the current page.
|
||||
|
||||
Args:
|
||||
full_page: When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport.
|
||||
|
||||
Returns:
|
||||
Base64 of the page screenshot
|
||||
"""
|
||||
page = await self.get_cur_page()
|
||||
|
||||
try:
|
||||
await page.bring_to_front()
|
||||
await page.wait_for_load_state(timeout=2000)
|
||||
except:
|
||||
logger.warning("bring to front load timeout")
|
||||
pass
|
||||
|
||||
screenshot = await page.screenshot(
|
||||
full_page=full_page,
|
||||
animations='disabled',
|
||||
timeout=600000
|
||||
)
|
||||
logger.info("page screenshot finished")
|
||||
screenshot_base64 = base64.b64encode(screenshot).decode('utf-8')
|
||||
return screenshot_base64
|
||||
|
||||
async def _get_observation(self, info: Dict[str, Any] = {}) -> Observation:
|
||||
fail_error = info.get('exception')
|
||||
if fail_error:
|
||||
return Observation(observer=self.name(), action_result=[ActionResult(error=fail_error)])
|
||||
|
||||
try:
|
||||
dom_tree = await self._parse_dom_tree()
|
||||
image = await self.screenshot()
|
||||
pixels_above, pixels_below = await self._scroll_info()
|
||||
info.update({"pixels_above": pixels_above,
|
||||
"pixels_below": pixels_below,
|
||||
"url": self.page.url})
|
||||
return Observation(observer=self.name(), dom_tree=dom_tree, image=image, info=info)
|
||||
except Exception as e:
|
||||
try:
|
||||
try:
|
||||
await self.page.go_back()
|
||||
except:
|
||||
logger.warning("current page abnormal, new page to use.")
|
||||
self.page = await self.browser_context.new_page()
|
||||
dom_tree = await self._parse_dom_tree()
|
||||
image = await self.screenshot()
|
||||
pixels_above, pixels_below = await self._scroll_info()
|
||||
info.update({"pixels_above": pixels_above,
|
||||
"pixels_below": pixels_below,
|
||||
"url": self.page.url})
|
||||
return Observation(observer=self.name(), dom_tree=dom_tree, image=image, info=info)
|
||||
except Exception as e:
|
||||
logger.warning(f"build observation fail, {traceback.format_exc()}")
|
||||
return Observation(observer=self.name(), action_result=[ActionResult(error=traceback.format_exc())])
|
||||
|
||||
async def _parse_dom_tree(self) -> DomTree:
|
||||
args = {
|
||||
'doHighlightElements': self.conf.get("do_highlight", True),
|
||||
'focusHighlightIndex': self.conf.get("focus_highlight", -1),
|
||||
'viewportExpansion': self.conf.get("viewport_expansion", 0),
|
||||
'debugMode': logger.getEffectiveLevel() == 10,
|
||||
}
|
||||
element_tree, element_map = await async_build_dom_tree(self.page, self.js_code, args)
|
||||
return DomTree(element_tree=element_tree, element_map=element_map)
|
||||
|
||||
async def _scroll_info(self) -> tuple[int, int]:
|
||||
"""Get scroll position information for the current page."""
|
||||
scroll_y = await self.page.evaluate('window.scrollY')
|
||||
viewport_height = await self.page.evaluate('window.innerHeight')
|
||||
total_height = await self.page.evaluate('document.documentElement.scrollHeight')
|
||||
pixels_above = scroll_y
|
||||
pixels_below = total_height - (scroll_y + viewport_height)
|
||||
return pixels_above, pixels_below
|
||||
|
||||
async def reset(self, *, seed: int | None = None, options: Dict[str, str] | None = None) -> Tuple[
|
||||
Observation, Dict[str, Any]]:
|
||||
await super().reset(seed=seed, options=options)
|
||||
if self.initialized:
|
||||
observation = await self._get_observation()
|
||||
observation.action_result = [ActionResult(content='start', keep=True)]
|
||||
self.cur_observation = observation
|
||||
return observation, {}
|
||||
|
||||
await self.close()
|
||||
await self.init()
|
||||
|
||||
if self.sleep_after_init > 0:
|
||||
await asyncio.sleep(self.sleep_after_init)
|
||||
|
||||
observation = await self._get_observation()
|
||||
observation.action_result = [ActionResult(content='start', keep=True)]
|
||||
observation.ability = ''
|
||||
self.cur_observation = observation
|
||||
return observation, {}
|
||||
|
||||
async def save_trace(self, trace_path: str | Path) -> None:
|
||||
if self.record_trace:
|
||||
await self.browser_context.tracing.stop(path=trace_path)
|
||||
|
||||
@property
|
||||
async def finished(self) -> bool:
|
||||
return self._finish
|
||||
|
||||
async def close(self) -> None:
|
||||
if hasattr(self, 'context') and self.browser_context:
|
||||
await self.browser_context.close()
|
||||
if hasattr(self, 'browser') and self.browser:
|
||||
await self.browser.close()
|
||||
if hasattr(self, 'playwright') and self.playwright:
|
||||
await self.playwright.stop()
|
||||
if self.initialized:
|
||||
await self.context_manager.__aexit__()
|
||||
|
||||
async def do_step(self, action: List[ActionModel], **kwargs) -> Tuple[
|
||||
Observation, float, bool, bool, Dict[str, Any]]:
|
||||
if not self.initialized:
|
||||
raise RuntimeError("Call init first before calling step.")
|
||||
|
||||
if not action:
|
||||
logger.warning(f"{self.name()} has no action")
|
||||
return build_observation(observer=self.name(), ability='', content='no action'), 0., False, False, {}
|
||||
|
||||
reward = 0
|
||||
fail_error = ""
|
||||
action_result = None
|
||||
|
||||
invalid_acts: List[int] = []
|
||||
for i, act in enumerate(action):
|
||||
if act.tool_name != 'browser':
|
||||
logger.warning(f"tool {act.tool_name} is not a browser!")
|
||||
invalid_acts.append(i)
|
||||
|
||||
if invalid_acts:
|
||||
for i in invalid_acts:
|
||||
action[i] = None
|
||||
|
||||
try:
|
||||
action_result, self.page = await self.action_executor.async_execute_action(action,
|
||||
observation=self.cur_observation,
|
||||
llm_config=self.conf.llm_config,
|
||||
**kwargs)
|
||||
reward = 1
|
||||
except Exception as e:
|
||||
fail_error = str(e)
|
||||
|
||||
info = {"exception": fail_error}
|
||||
terminated = kwargs.get("terminated", False)
|
||||
for res in action_result:
|
||||
if res.is_done:
|
||||
terminated = res.is_done
|
||||
info['done'] = True
|
||||
self._finish = True
|
||||
if res.error:
|
||||
fail_error += res.error
|
||||
|
||||
contains_write_to_file = any(act.action_name == BrowserAction.WRITE_TO_FILE.value.name for act in action if act)
|
||||
if contains_write_to_file:
|
||||
msg = ""
|
||||
for action_result_elem in action_result:
|
||||
msg = action_result_elem.content
|
||||
# write_to_file observation
|
||||
return (Observation(content=msg, action_result=action_result, info=info),
|
||||
reward,
|
||||
terminated,
|
||||
kwargs.get("truncated", False),
|
||||
info)
|
||||
elif fail_error:
|
||||
# failed error observation
|
||||
return (Observation(action_result=action_result, observer=self.name()),
|
||||
reward,
|
||||
terminated,
|
||||
kwargs.get("truncated", False),
|
||||
info)
|
||||
else:
|
||||
# normal observation
|
||||
observation = await self._get_observation(info)
|
||||
observation.action_result = action_result
|
||||
observation.ability = action[-1].action_name
|
||||
self.cur_observation = observation
|
||||
return (observation,
|
||||
reward,
|
||||
terminated,
|
||||
kwargs.get("truncated", False),
|
||||
info)
|
||||
@@ -0,0 +1,368 @@
|
||||
# coding: utf-8
|
||||
# Copyright (c) 2025 inclusionAI.
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import traceback
|
||||
from importlib import resources
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Tuple, List, Union
|
||||
|
||||
from aworld.config import ConfigDict
|
||||
from examples.common.tools.common import package
|
||||
from examples.common.tools.tool_action import BrowserAction
|
||||
from aworld.core.common import Observation, ActionModel, ActionResult
|
||||
from aworld.logs.util import logger
|
||||
from aworld.core.tool.base import action_executor, ToolFactory
|
||||
from aworld.core.tool.base import Tool
|
||||
from aworld.utils.import_package import is_package_installed
|
||||
from examples.common.tools.browsers.action.executor import BrowserToolActionExecutor
|
||||
from examples.common.tools.browsers.util.dom import DomTree
|
||||
from examples.common.tools.conf import BrowserToolConfig
|
||||
from examples.common.tools.browsers.util.dom_build import build_dom_tree
|
||||
from aworld.utils import import_package
|
||||
from aworld.tools.utils import build_observation
|
||||
|
||||
URL_MAX_LENGTH = 4096
|
||||
UTF8 = "".join(chr(x) for x in range(0, 55290))
|
||||
ASCII = "".join(chr(x) for x in range(32, 128))
|
||||
|
||||
BROWSER = "browser"
|
||||
|
||||
|
||||
@ToolFactory.register(name=BROWSER,
|
||||
desc="browser",
|
||||
supported_action=BrowserAction,
|
||||
conf_file_name=f'browser_tool.yaml',
|
||||
dir=f"{Path(__file__).parent.absolute()}")
|
||||
class BrowserTool(Tool):
|
||||
def __init__(self, conf: Union[ConfigDict, BrowserToolConfig], **kwargs) -> None:
|
||||
super(BrowserTool, self).__init__(conf, **kwargs)
|
||||
|
||||
self.initialized = False
|
||||
self._finish = False
|
||||
self.record_trace = self.conf.get("enable_recording", False)
|
||||
self.sleep_after_init = self.conf.get("sleep_after_init", False)
|
||||
|
||||
dom_js_path = self.conf.get('dom_js_path')
|
||||
if dom_js_path and os.path.exists(dom_js_path):
|
||||
with open(dom_js_path, 'r') as read:
|
||||
self.js_code = read.read()
|
||||
else:
|
||||
self.js_code = resources.read_text(f'{package}.browsers.script',
|
||||
'buildDomTree.js')
|
||||
self.cur_observation = None
|
||||
if not is_package_installed('playwright'):
|
||||
import_package("playwright")
|
||||
logger.info("playwright install...")
|
||||
try:
|
||||
subprocess.check_call('playwright install', shell=True, timeout=300)
|
||||
except Exception as e:
|
||||
logger.error(f"Fail to auto execute playwright install, you can install manually\n {e}")
|
||||
|
||||
def init(self) -> None:
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
if self.initialized:
|
||||
return
|
||||
|
||||
self.context_manager = sync_playwright()
|
||||
self.playwright = self.context_manager.start()
|
||||
self.browser = self._create_browser()
|
||||
self.browser_context = self._create_browser_context()
|
||||
|
||||
if self.record_trace:
|
||||
self.browser_context.tracing.start(screenshots=True, snapshots=True)
|
||||
|
||||
self.page = self.browser_context.new_page()
|
||||
if self.conf.get("custom_executor"):
|
||||
self.action_executor = BrowserToolActionExecutor(self)
|
||||
else:
|
||||
self.action_executor = action_executor
|
||||
self.initialized = True
|
||||
|
||||
def _create_browser(self):
|
||||
browse_name = self.conf.get("browse_name", "chromium")
|
||||
browse = getattr(self.playwright, browse_name)
|
||||
cdp_url = self.conf.get("cdp_url")
|
||||
wss_url = self.conf.get("wss_url")
|
||||
if cdp_url:
|
||||
if browse_name != "chromium":
|
||||
logger.warning(f"{browse_name} unsupported CDP, will use chromium browser")
|
||||
browse = self.playwright.chromium
|
||||
logger.info(f"Connecting to remote browser via CDP {cdp_url}")
|
||||
browser = browse.connect_over_cdp(cdp_url)
|
||||
elif wss_url:
|
||||
logger.info(f"Connecting to remote browser via wss {wss_url}")
|
||||
browser = browse.connect(wss_url)
|
||||
else:
|
||||
headless = self.conf.get("headless", False)
|
||||
slow_mo = self.conf.get("slow_mo", 0)
|
||||
disable_security_args = []
|
||||
if self.conf.get('disable_security', False):
|
||||
disable_security_args = ['--disable-web-security',
|
||||
'--disable-site-isolation-trials',
|
||||
'--disable-features=IsolateOrigins,site-per-process']
|
||||
args = ['--no-sandbox',
|
||||
'--disable-crash-reporte',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-infobars',
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-popup-blocking',
|
||||
'--disable-backgrounding-occluded-windows',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--disable-window-activation',
|
||||
'--disable-focus-on-load',
|
||||
'--no-first-run',
|
||||
'--no-default-browser-check',
|
||||
'--no-startup-window',
|
||||
'--window-position=0,0',
|
||||
'--window-size=1280,720'] + disable_security_args
|
||||
browser = browse.launch(
|
||||
headless=headless,
|
||||
slow_mo=slow_mo,
|
||||
args=args,
|
||||
proxy=self.conf.get('proxy'),
|
||||
)
|
||||
return browser
|
||||
|
||||
def _create_browser_context(self):
|
||||
"""Creates a new browser context with anti-detection measures and loads cookies if available."""
|
||||
from playwright.sync_api import ViewportSize
|
||||
|
||||
browser = self.browser
|
||||
if self.conf.get("cdp_url") and len(browser.contexts) > 0:
|
||||
context = browser.contexts[0]
|
||||
else:
|
||||
viewport_size = ViewportSize(width=self.conf.get("width", 1280),
|
||||
height=self.conf.get("height", 720))
|
||||
disable_security = self.conf.get('disable_security', False)
|
||||
|
||||
context = browser.new_context(viewport=viewport_size,
|
||||
no_viewport=False,
|
||||
user_agent=self.conf.get('user_agent'),
|
||||
java_script_enabled=True,
|
||||
bypass_csp=disable_security,
|
||||
ignore_https_errors=disable_security,
|
||||
record_video_dir=self.conf.get('working_dir'),
|
||||
record_video_size=viewport_size,
|
||||
locale=self.conf.get('locale'),
|
||||
storage_state=self.conf.get("storage_state", None),
|
||||
geolocation=self.conf.get("geolocation", None),
|
||||
device_scale_factor=1)
|
||||
if "chromium" == self.conf.get("browse_name", "chromium"):
|
||||
context.grant_permissions(['camera', 'microphone'])
|
||||
|
||||
if self.conf.get('working_dir'):
|
||||
context.tracing.start(screenshots=True, snapshots=True, sources=True)
|
||||
|
||||
cookie_file = self.conf.get('cookies_file')
|
||||
if cookie_file and os.path.exists(cookie_file):
|
||||
with open(cookie_file, 'r') as read:
|
||||
cookies = json.loads(read.read())
|
||||
context.add_cookies(cookies)
|
||||
logger.info(f'Cookies load from {cookie_file} finished')
|
||||
|
||||
if self.conf.get('private'):
|
||||
js = resources.read_text(f"{package}.browsers.script", "stealth.min.js")
|
||||
context.add_init_script(js)
|
||||
|
||||
return context
|
||||
|
||||
def get_cur_page(self):
|
||||
return self.page
|
||||
|
||||
def screenshot(self, full_page: bool = False) -> str:
|
||||
"""Returns a base64 encoded screenshot of the current page.
|
||||
|
||||
Args:
|
||||
full_page: When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport.
|
||||
|
||||
Returns:
|
||||
Base64 of the page screenshot
|
||||
"""
|
||||
page = self.get_cur_page()
|
||||
|
||||
try:
|
||||
page.bring_to_front()
|
||||
page.wait_for_load_state(timeout=2000)
|
||||
except:
|
||||
logger.warning("bring to front load timeout")
|
||||
pass
|
||||
|
||||
screenshot = page.screenshot(
|
||||
full_page=full_page,
|
||||
animations='disabled',
|
||||
timeout=600000
|
||||
)
|
||||
logger.info("page screenshot finished")
|
||||
screenshot_base64 = base64.b64encode(screenshot).decode('utf-8')
|
||||
return screenshot_base64
|
||||
|
||||
def _get_observation(self, info: Dict[str, Any] = {}) -> Observation:
|
||||
fail_error = info.get('exception')
|
||||
if fail_error:
|
||||
return Observation(observer=self.name(), action_result=[ActionResult(error=fail_error)])
|
||||
|
||||
try:
|
||||
dom_tree = self._parse_dom_tree()
|
||||
image = self.screenshot()
|
||||
pixels_above, pixels_below = self._scroll_info()
|
||||
info.update({"pixels_above": pixels_above,
|
||||
"pixels_below": pixels_below,
|
||||
"url": self.page.url})
|
||||
return Observation(observer=self.name(),
|
||||
dom_tree=dom_tree,
|
||||
image=image,
|
||||
info=info)
|
||||
except Exception as e:
|
||||
try:
|
||||
self.page.go_back()
|
||||
except:
|
||||
logger.warning("current page abnormal, new page to use.")
|
||||
self.page = self.browser_context.new_page()
|
||||
try:
|
||||
dom_tree = self._parse_dom_tree()
|
||||
image = self.screenshot()
|
||||
pixels_above, pixels_below = self._scroll_info()
|
||||
info.update({"pixels_above": pixels_above,
|
||||
"pixels_below": pixels_below,
|
||||
"url": self.page.url})
|
||||
return Observation(observer=self.name(), dom_tree=dom_tree, image=image, info=info)
|
||||
except Exception as e:
|
||||
logger.warning(f"build observation fail, {traceback.format_exc()}")
|
||||
return Observation(observer=self.name(), action_result=[ActionResult(error=traceback.format_exc())])
|
||||
|
||||
def _parse_dom_tree(self) -> DomTree:
|
||||
args = {
|
||||
'doHighlightElements': self.conf.get("do_highlight", True),
|
||||
'focusHighlightIndex': self.conf.get("focus_highlight", -1),
|
||||
'viewportExpansion': self.conf.get("viewport_expansion", 0),
|
||||
'debugMode': logger.getEffectiveLevel() == 10,
|
||||
}
|
||||
element_tree, element_map = build_dom_tree(self.page, self.js_code, args)
|
||||
return DomTree(element_tree=element_tree, element_map=element_map)
|
||||
|
||||
def _scroll_info(self) -> tuple[int, int]:
|
||||
"""Get scroll position information for the current page."""
|
||||
scroll_y = self.page.evaluate('window.scrollY')
|
||||
viewport_height = self.page.evaluate('window.innerHeight')
|
||||
total_height = self.page.evaluate('document.documentElement.scrollHeight')
|
||||
pixels_above = scroll_y
|
||||
pixels_below = total_height - (scroll_y + viewport_height)
|
||||
return pixels_above, pixels_below
|
||||
|
||||
def reset(self, *, seed: int | None = None, options: Dict[str, str] | None = None) -> Tuple[
|
||||
Observation, Dict[str, Any]]:
|
||||
super().reset(seed=seed, options=options)
|
||||
if self.initialized:
|
||||
observation = self._get_observation()
|
||||
observation.action_result = [ActionResult(content='start', keep=True)]
|
||||
self.cur_observation = observation
|
||||
return observation, {}
|
||||
|
||||
self.close()
|
||||
self.init()
|
||||
|
||||
if self.sleep_after_init > 0:
|
||||
time.sleep(self.sleep_after_init)
|
||||
|
||||
observation = self._get_observation()
|
||||
observation.action_result = [ActionResult(content='start', keep=True)]
|
||||
self.cur_observation = observation
|
||||
return observation, {}
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self._finish
|
||||
|
||||
def save_trace(self, trace_path: str | Path) -> None:
|
||||
if self.record_trace:
|
||||
self.browser_context.tracing.stop(path=trace_path)
|
||||
|
||||
def close(self) -> None:
|
||||
if hasattr(self, 'context') and self.browser_context:
|
||||
self.browser_context.close()
|
||||
if hasattr(self, 'browser') and self.browser:
|
||||
self.browser.close()
|
||||
if hasattr(self, 'playwright') and self.playwright:
|
||||
self.playwright.stop()
|
||||
|
||||
if self.initialized:
|
||||
self.context_manager.__exit__()
|
||||
|
||||
def do_step(self, action: List[ActionModel], **kwargs) -> Tuple[
|
||||
Observation, float, bool, bool, Dict[str, Any]]:
|
||||
if not self.initialized:
|
||||
raise RuntimeError("Call init first before calling step.")
|
||||
|
||||
if not action:
|
||||
logger.warning(f"{self.name()} has no action")
|
||||
return build_observation(observer=self.name(), ability='', content='no action'), 0., False, False, {}
|
||||
|
||||
reward = 0
|
||||
fail_error = ""
|
||||
action_result = None
|
||||
|
||||
invalid_acts: List[int] = []
|
||||
for i, act in enumerate(action):
|
||||
if act.tool_name != BROWSER:
|
||||
logger.warning(f"tool {act.tool_name} is not a browser!")
|
||||
invalid_acts.append(i)
|
||||
|
||||
if invalid_acts:
|
||||
for i in invalid_acts:
|
||||
action[i] = None
|
||||
|
||||
try:
|
||||
action_result, self.page = self.action_executor.execute_action(action,
|
||||
observation=self.cur_observation,
|
||||
llm_config=self.conf.llm_config,
|
||||
**kwargs)
|
||||
reward = 1
|
||||
except Exception as e:
|
||||
fail_error = str(e)
|
||||
|
||||
info = {"exception": fail_error}
|
||||
terminated = kwargs.get("terminated", False)
|
||||
if action_result:
|
||||
for res in action_result:
|
||||
if res.is_done:
|
||||
terminated = res.is_done
|
||||
info['done'] = True
|
||||
self._finish = True
|
||||
if res.error:
|
||||
fail_error += res.error
|
||||
|
||||
contains_write_to_file = any(act.action_name == BrowserAction.WRITE_TO_FILE.value.name for act in action if act)
|
||||
if contains_write_to_file:
|
||||
msg = ""
|
||||
for action_result_elem in action_result:
|
||||
msg = action_result_elem.content
|
||||
# write_to_file observation
|
||||
return (Observation(content=msg, action_result=action_result, info=info),
|
||||
reward,
|
||||
terminated,
|
||||
kwargs.get("truncated", False),
|
||||
info)
|
||||
elif fail_error:
|
||||
# failed error observation
|
||||
return (Observation(action_result=action_result, observer=self.name()),
|
||||
reward,
|
||||
terminated,
|
||||
kwargs.get("truncated", False),
|
||||
info)
|
||||
else:
|
||||
# normal observation
|
||||
observation = self._get_observation(info)
|
||||
observation.ability = action[-1].action_name
|
||||
observation.action_result = action_result
|
||||
self.cur_observation = observation
|
||||
return (observation,
|
||||
reward,
|
||||
terminated,
|
||||
kwargs.get("truncated", False),
|
||||
info)
|
||||
@@ -0,0 +1,24 @@
|
||||
browse_name: chromium
|
||||
headless: False
|
||||
width: 1280
|
||||
height: 720
|
||||
slow_mo: 0
|
||||
disable_security: False
|
||||
custom_executor: False
|
||||
dom_js_path:
|
||||
private:
|
||||
locale:
|
||||
geolocation:
|
||||
storage_state:
|
||||
do_highlight: True
|
||||
focus_highlight: -1
|
||||
viewport_expansion: 0
|
||||
cdp_url:
|
||||
wss_url:
|
||||
proxy:
|
||||
cookies_file:
|
||||
working_dir:
|
||||
enable_recording: False
|
||||
sleep_after_init: 0
|
||||
max_retry: 3
|
||||
reuse: True
|
||||
@@ -0,0 +1,2 @@
|
||||
playwright
|
||||
markdownify
|
||||
+1056
File diff suppressed because it is too large
Load Diff
+7
File diff suppressed because one or more lines are too long
@@ -0,0 +1,210 @@
|
||||
# coding: utf-8
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Dict, List
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class Coordinates(BaseModel):
|
||||
x: int
|
||||
y: int
|
||||
|
||||
|
||||
class CoordinateSet(BaseModel):
|
||||
top_left: Coordinates
|
||||
top_right: Coordinates
|
||||
bottom_left: Coordinates
|
||||
bottom_right: Coordinates
|
||||
center: Coordinates
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
class ViewportInfo(BaseModel):
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class HashedDomElement:
|
||||
"""
|
||||
Hash of the dom element to be used as a unique identifier
|
||||
"""
|
||||
|
||||
branch_path_hash: str
|
||||
attributes_hash: str
|
||||
xpath_hash: str
|
||||
|
||||
|
||||
@dataclass(frozen=False)
|
||||
class DOMBaseNode:
|
||||
is_visible: bool
|
||||
# Use None as default and set parent later to avoid circular reference issues
|
||||
parent: Optional['DOMElementNode']
|
||||
|
||||
|
||||
@dataclass(frozen=False)
|
||||
class DOMTextNode(DOMBaseNode):
|
||||
text: str
|
||||
type: str = 'TEXT_NODE'
|
||||
|
||||
def has_parent_with_highlight_index(self) -> bool:
|
||||
current = self.parent
|
||||
while current is not None:
|
||||
# stop if the element has a highlight index (will be handled separately)
|
||||
if current.highlight_index is not None:
|
||||
return True
|
||||
|
||||
current = current.parent
|
||||
return False
|
||||
|
||||
def is_parent_in_viewport(self) -> bool:
|
||||
if self.parent is None:
|
||||
return False
|
||||
return self.parent.is_in_viewport
|
||||
|
||||
def is_parent_top_element(self) -> bool:
|
||||
if self.parent is None:
|
||||
return False
|
||||
return self.parent.is_top_element
|
||||
|
||||
|
||||
@dataclass(frozen=False)
|
||||
class DOMElementNode(DOMBaseNode):
|
||||
"""
|
||||
xpath: the xpath of the element from the last root node (shadow root or iframe OR document if no shadow root or iframe).
|
||||
To properly reference the element we need to recursively switch the root node until we find the element (work you way up the tree with `.parent`)
|
||||
"""
|
||||
|
||||
tag_name: str
|
||||
xpath: str
|
||||
attributes: Dict[str, str]
|
||||
children: List[DOMBaseNode]
|
||||
is_interactive: bool = False
|
||||
is_top_element: bool = False
|
||||
is_in_viewport: bool = False
|
||||
shadow_root: bool = False
|
||||
highlight_index: Optional[int] = None
|
||||
viewport_coordinates: Optional[CoordinateSet] = None
|
||||
page_coordinates: Optional[CoordinateSet] = None
|
||||
viewport_info: Optional[ViewportInfo] = None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
tag_str = f'<{self.tag_name}'
|
||||
|
||||
# Add attributes
|
||||
for key, value in self.attributes.items():
|
||||
tag_str += f' {key}="{value}"'
|
||||
tag_str += '>'
|
||||
|
||||
# Add extra info
|
||||
extras = []
|
||||
if self.is_interactive:
|
||||
extras.append('interactive')
|
||||
if self.is_top_element:
|
||||
extras.append('top')
|
||||
if self.shadow_root:
|
||||
extras.append('shadow-root')
|
||||
if self.highlight_index is not None:
|
||||
extras.append(f'highlight:{self.highlight_index}')
|
||||
if self.is_in_viewport:
|
||||
extras.append('in-viewport')
|
||||
|
||||
if extras:
|
||||
tag_str += f' [{", ".join(extras)}]'
|
||||
|
||||
return tag_str
|
||||
|
||||
def get_all_text_till_next_clickable_element(self, max_depth: int = -1) -> str:
|
||||
text_parts = []
|
||||
|
||||
def collect_text(node: DOMBaseNode, current_depth: int) -> None:
|
||||
if max_depth != -1 and current_depth > max_depth:
|
||||
return
|
||||
|
||||
# Skip this branch if we hit a highlighted element (except for the current node)
|
||||
if isinstance(node, DOMElementNode) and node != self and node.highlight_index is not None:
|
||||
return
|
||||
|
||||
if isinstance(node, DOMTextNode):
|
||||
text_parts.append(node.text)
|
||||
elif isinstance(node, DOMElementNode):
|
||||
for child in node.children:
|
||||
collect_text(child, current_depth + 1)
|
||||
|
||||
collect_text(self, 0)
|
||||
return '\n'.join(text_parts).strip()
|
||||
|
||||
def clickable_elements_to_string(self, include_attributes: list[str] | None = None) -> str:
|
||||
"""Convert the processed DOM content to HTML."""
|
||||
formatted_text = []
|
||||
|
||||
def process_node(node: DOMBaseNode, depth: int) -> None:
|
||||
if isinstance(node, DOMElementNode):
|
||||
# Add element with highlight_index
|
||||
if node.highlight_index is not None:
|
||||
attributes_str = ''
|
||||
text = node.get_all_text_till_next_clickable_element()
|
||||
if include_attributes:
|
||||
attributes = list(
|
||||
set(
|
||||
[
|
||||
str(value)
|
||||
for key, value in node.attributes.items()
|
||||
if key in include_attributes and value != node.tag_name
|
||||
]
|
||||
)
|
||||
)
|
||||
if text in attributes:
|
||||
attributes.remove(text)
|
||||
attributes_str = ';'.join(attributes)
|
||||
line = f'[{node.highlight_index}]<{node.tag_name} '
|
||||
if attributes_str:
|
||||
line += f'{attributes_str}'
|
||||
if text:
|
||||
if attributes_str:
|
||||
line += f'>{text}'
|
||||
else:
|
||||
line += f'{text}'
|
||||
line += '/>'
|
||||
formatted_text.append(line)
|
||||
|
||||
# Process children regardless
|
||||
for child in node.children:
|
||||
process_node(child, depth + 1)
|
||||
|
||||
elif isinstance(node, DOMTextNode):
|
||||
# Add text only if it doesn't have a highlighted parent
|
||||
if not node.has_parent_with_highlight_index() and node.is_visible: # and node.is_parent_top_element()
|
||||
formatted_text.append(f'{node.text}')
|
||||
|
||||
process_node(self, 0)
|
||||
return '\n'.join(formatted_text)
|
||||
|
||||
def get_file_upload_element(self, check_siblings: bool = True) -> Optional['DOMElementNode']:
|
||||
# Check if current element is a file input
|
||||
if self.tag_name == 'input' and self.attributes.get('type') == 'file':
|
||||
return self
|
||||
|
||||
# Check children
|
||||
for child in self.children:
|
||||
if isinstance(child, DOMElementNode):
|
||||
result = child.get_file_upload_element(check_siblings=False)
|
||||
if result:
|
||||
return result
|
||||
|
||||
# Check siblings only for the initial call
|
||||
if check_siblings and self.parent:
|
||||
for sibling in self.parent.children:
|
||||
if sibling is not self and isinstance(sibling, DOMElementNode):
|
||||
result = sibling.get_file_upload_element(check_siblings=False)
|
||||
if result:
|
||||
return result
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class DomTree(BaseModel):
|
||||
element_tree: DOMElementNode
|
||||
element_map: Dict[int, DOMElementNode]
|
||||
@@ -0,0 +1,138 @@
|
||||
# coding: utf-8
|
||||
|
||||
# Derived from browser_use DomService, we use it as a utility method, and supports sync and async.
|
||||
|
||||
import gc
|
||||
import json
|
||||
|
||||
from typing import Dict, Any, Tuple, Optional
|
||||
|
||||
from aworld.utils.async_func import async_func
|
||||
from examples.common.tools.browsers.util.dom import DOMElementNode, DOMBaseNode, DOMTextNode, ViewportInfo
|
||||
from aworld.logs.util import logger
|
||||
|
||||
|
||||
async def async_build_dom_tree(page, js_code: str, args: Dict[str, Any]) -> Tuple[DOMElementNode, Dict[int, DOMElementNode]]:
|
||||
if await page.evaluate('1+1') != 2:
|
||||
raise ValueError('The page cannot evaluate javascript code properly')
|
||||
|
||||
# NOTE: We execute JS code in the browser to extract important DOM information.
|
||||
# The returned hash map contains information about the DOM tree and the
|
||||
# relationship between the DOM elements.
|
||||
try:
|
||||
eval_page = await page.evaluate(js_code, args)
|
||||
except Exception as e:
|
||||
logger.error('Error evaluating JavaScript: %s', e)
|
||||
raise
|
||||
|
||||
# Only log performance metrics in debug mode
|
||||
if args.get("debugMode") and 'perfMetrics' in eval_page:
|
||||
logger.debug('DOM Tree Building Performance Metrics:\n%s', json.dumps(eval_page['perfMetrics'], indent=2))
|
||||
|
||||
return await async_func(_construct_dom_tree)(eval_page)
|
||||
|
||||
|
||||
def build_dom_tree(page, js_code: str, args: Dict[str, Any]) -> Tuple[DOMElementNode, Dict[int, DOMElementNode]]:
|
||||
if page.evaluate('1+1') != 2:
|
||||
raise ValueError('The page cannot evaluate javascript code properly')
|
||||
|
||||
# NOTE: We execute JS code in the browser to extract important DOM information.
|
||||
# The returned hash map contains information about the DOM tree and the
|
||||
# relationship between the DOM elements.
|
||||
try:
|
||||
eval_page = page.evaluate(js_code, args)
|
||||
except Exception as e:
|
||||
logger.error('Error evaluating JavaScript: %s', e)
|
||||
raise
|
||||
|
||||
# Only log performance metrics in debug mode
|
||||
if args.get("debugMode") and 'perfMetrics' in eval_page:
|
||||
logger.debug('DOM Tree Building Performance Metrics:\n%s', json.dumps(eval_page['perfMetrics'], indent=2))
|
||||
|
||||
return _construct_dom_tree(eval_page)
|
||||
|
||||
|
||||
def _construct_dom_tree(eval_page: dict, ) -> tuple[DOMElementNode, Dict[int, DOMElementNode]]:
|
||||
js_node_map = eval_page['map']
|
||||
js_root_id = eval_page['rootId']
|
||||
|
||||
selector_map = {}
|
||||
node_map = {}
|
||||
|
||||
for id, node_data in js_node_map.items():
|
||||
node, children_ids = _parse_node(node_data)
|
||||
if node is None:
|
||||
continue
|
||||
|
||||
node_map[id] = node
|
||||
|
||||
if isinstance(node, DOMElementNode) and node.highlight_index is not None:
|
||||
selector_map[node.highlight_index] = node
|
||||
|
||||
# NOTE: We know that we are building the tree bottom up
|
||||
# and all children are already processed.
|
||||
if isinstance(node, DOMElementNode):
|
||||
for child_id in children_ids:
|
||||
if child_id not in node_map:
|
||||
continue
|
||||
|
||||
child_node = node_map[child_id]
|
||||
|
||||
child_node.parent = node
|
||||
node.children.append(child_node)
|
||||
|
||||
html_to_dict = node_map[str(js_root_id)]
|
||||
|
||||
del node_map
|
||||
del js_node_map
|
||||
del js_root_id
|
||||
|
||||
gc.collect()
|
||||
|
||||
if html_to_dict is None or not isinstance(html_to_dict, DOMElementNode):
|
||||
raise ValueError('Failed to parse HTML to dictionary')
|
||||
|
||||
return html_to_dict, selector_map
|
||||
|
||||
|
||||
def _parse_node(node_data: dict, ) -> Tuple[Optional[DOMBaseNode], list[int]]:
|
||||
if not node_data:
|
||||
return None, []
|
||||
|
||||
# Process text nodes immediately
|
||||
if node_data.get('type') == 'TEXT_NODE':
|
||||
text_node = DOMTextNode(
|
||||
text=node_data['text'],
|
||||
is_visible=node_data['isVisible'],
|
||||
parent=None,
|
||||
)
|
||||
return text_node, []
|
||||
|
||||
# Process coordinates if they exist for element nodes
|
||||
|
||||
viewport_info = None
|
||||
|
||||
if 'viewport' in node_data:
|
||||
viewport_info = ViewportInfo(
|
||||
width=node_data['viewport']['width'],
|
||||
height=node_data['viewport']['height'],
|
||||
)
|
||||
|
||||
element_node = DOMElementNode(
|
||||
tag_name=node_data['tagName'],
|
||||
xpath=node_data['xpath'],
|
||||
attributes=node_data.get('attributes', {}),
|
||||
children=[],
|
||||
is_visible=node_data.get('isVisible', False),
|
||||
is_interactive=node_data.get('isInteractive', False),
|
||||
is_top_element=node_data.get('isTopElement', False),
|
||||
is_in_viewport=node_data.get('isInViewport', False),
|
||||
highlight_index=node_data.get('highlightIndex'),
|
||||
shadow_root=node_data.get('shadowRoot', False),
|
||||
parent=None,
|
||||
viewport_info=viewport_info,
|
||||
)
|
||||
|
||||
children_ids = node_data.get('children', [])
|
||||
|
||||
return element_node, children_ids
|
||||
Reference in New Issue
Block a user