ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,282 @@
|
||||
import base64
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import anyio
|
||||
from bubus import BaseEvent
|
||||
from pydantic import Field, field_validator
|
||||
from uuid_extensions import uuid7str
|
||||
|
||||
MAX_STRING_LENGTH = 100000 # 100K chars ~ 25k tokens should be enough
|
||||
MAX_URL_LENGTH = 100000
|
||||
MAX_TASK_LENGTH = 100000
|
||||
MAX_COMMENT_LENGTH = 2000
|
||||
MAX_FILE_CONTENT_SIZE = 50 * 1024 * 1024 # 50MB
|
||||
|
||||
|
||||
class UpdateAgentTaskEvent(BaseEvent):
|
||||
# Required fields for identification
|
||||
id: str # The task ID to update
|
||||
user_id: str = Field(max_length=255) # For authorization
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
|
||||
# Optional fields that can be updated
|
||||
stopped: bool | None = None
|
||||
paused: bool | None = None
|
||||
done_output: str | None = Field(None, max_length=MAX_STRING_LENGTH)
|
||||
finished_at: datetime | None = None
|
||||
agent_state: dict | None = None
|
||||
user_feedback_type: str | None = Field(None, max_length=10) # UserFeedbackType enum value as string
|
||||
user_comment: str | None = Field(None, max_length=MAX_COMMENT_LENGTH)
|
||||
gif_url: str | None = Field(None, max_length=MAX_URL_LENGTH)
|
||||
|
||||
@classmethod
|
||||
def from_agent(cls, agent) -> 'UpdateAgentTaskEvent':
|
||||
"""Create an UpdateAgentTaskEvent from an Agent instance"""
|
||||
if not hasattr(agent, '_task_start_time'):
|
||||
raise ValueError('Agent must have _task_start_time attribute')
|
||||
|
||||
done_output = agent.history.final_result() if agent.history else None
|
||||
return cls(
|
||||
id=str(agent.task_id),
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
stopped=agent.state.stopped if hasattr(agent.state, 'stopped') else False,
|
||||
paused=agent.state.paused if hasattr(agent.state, 'paused') else False,
|
||||
done_output=done_output,
|
||||
finished_at=datetime.now(timezone.utc) if agent.history and agent.history.is_done() else None,
|
||||
agent_state=agent.state.model_dump() if hasattr(agent.state, 'model_dump') else {},
|
||||
user_feedback_type=None,
|
||||
user_comment=None,
|
||||
gif_url=None,
|
||||
# user_feedback_type and user_comment would be set by the API/frontend
|
||||
# gif_url would be set after GIF generation if needed
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentOutputFileEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255)
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
task_id: str
|
||||
file_name: str = Field(max_length=255)
|
||||
file_content: str | None = None # Base64 encoded file content
|
||||
content_type: str | None = Field(None, max_length=100) # MIME type for file uploads
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
|
||||
@field_validator('file_content')
|
||||
@classmethod
|
||||
def validate_file_size(cls, v: str | None) -> str | None:
|
||||
"""Validate base64 file content size."""
|
||||
if v is None:
|
||||
return v
|
||||
# Remove data URL prefix if present
|
||||
if ',' in v:
|
||||
v = v.split(',')[1]
|
||||
# Estimate decoded size (base64 is ~33% larger)
|
||||
estimated_size = len(v) * 3 / 4
|
||||
if estimated_size > MAX_FILE_CONTENT_SIZE:
|
||||
raise ValueError(f'File content exceeds maximum size of {MAX_FILE_CONTENT_SIZE / 1024 / 1024}MB')
|
||||
return v
|
||||
|
||||
@classmethod
|
||||
async def from_agent_and_file(cls, agent, output_path: str) -> 'CreateAgentOutputFileEvent':
|
||||
"""Create a CreateAgentOutputFileEvent from a file path"""
|
||||
|
||||
gif_path = Path(output_path)
|
||||
if not gif_path.exists():
|
||||
raise FileNotFoundError(f'File not found: {output_path}')
|
||||
|
||||
gif_size = os.path.getsize(gif_path)
|
||||
|
||||
# Read GIF content for base64 encoding if needed
|
||||
gif_content = None
|
||||
if gif_size < 50 * 1024 * 1024: # Only read if < 50MB
|
||||
async with await anyio.open_file(gif_path, 'rb') as f:
|
||||
gif_bytes = await f.read()
|
||||
gif_content = base64.b64encode(gif_bytes).decode('utf-8')
|
||||
|
||||
return cls(
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
task_id=str(agent.task_id),
|
||||
file_name=gif_path.name,
|
||||
file_content=gif_content, # Base64 encoded
|
||||
content_type='image/gif',
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentStepEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255) # Added for authorization checks
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
agent_task_id: str
|
||||
step: int
|
||||
evaluation_previous_goal: str = Field(max_length=MAX_STRING_LENGTH)
|
||||
memory: str = Field(max_length=MAX_STRING_LENGTH)
|
||||
next_goal: str = Field(max_length=MAX_STRING_LENGTH)
|
||||
actions: list[dict]
|
||||
screenshot_url: str | None = Field(None, max_length=MAX_FILE_CONTENT_SIZE) # ~50MB for base64 images
|
||||
url: str = Field(default='', max_length=MAX_URL_LENGTH)
|
||||
|
||||
@field_validator('screenshot_url')
|
||||
@classmethod
|
||||
def validate_screenshot_size(cls, v: str | None) -> str | None:
|
||||
"""Validate screenshot URL or base64 content size."""
|
||||
if v is None or not v.startswith('data:'):
|
||||
return v
|
||||
# It's base64 data, check size
|
||||
if ',' in v:
|
||||
base64_part = v.split(',')[1]
|
||||
estimated_size = len(base64_part) * 3 / 4
|
||||
if estimated_size > MAX_FILE_CONTENT_SIZE:
|
||||
raise ValueError(f'Screenshot content exceeds maximum size of {MAX_FILE_CONTENT_SIZE / 1024 / 1024}MB')
|
||||
return v
|
||||
|
||||
@classmethod
|
||||
def from_agent_step(
|
||||
cls, agent, model_output, result: list, actions_data: list[dict], browser_state_summary
|
||||
) -> 'CreateAgentStepEvent':
|
||||
"""Create a CreateAgentStepEvent from agent step data"""
|
||||
# Get first action details if available
|
||||
first_action = model_output.action[0] if model_output.action else None
|
||||
|
||||
# Extract current state from model output
|
||||
current_state = model_output.current_state if hasattr(model_output, 'current_state') else None
|
||||
|
||||
# Capture screenshot as base64 data URL if available
|
||||
screenshot_url = None
|
||||
if browser_state_summary.screenshot:
|
||||
screenshot_url = f'data:image/png;base64,{browser_state_summary.screenshot}'
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.debug(f'📸 Including screenshot in CreateAgentStepEvent, length: {len(browser_state_summary.screenshot)}')
|
||||
else:
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.debug('📸 No screenshot in browser_state_summary for CreateAgentStepEvent')
|
||||
|
||||
return cls(
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
agent_task_id=str(agent.task_id),
|
||||
step=agent.state.n_steps,
|
||||
evaluation_previous_goal=current_state.evaluation_previous_goal if current_state else '',
|
||||
memory=current_state.memory if current_state else '',
|
||||
next_goal=current_state.next_goal if current_state else '',
|
||||
actions=actions_data, # List of action dicts
|
||||
url=browser_state_summary.url,
|
||||
screenshot_url=screenshot_url,
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentTaskEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255) # Added for authorization checks
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
agent_session_id: str
|
||||
llm_model: str = Field(max_length=100) # LLMModel enum value as string
|
||||
stopped: bool = False
|
||||
paused: bool = False
|
||||
task: str = Field(max_length=MAX_TASK_LENGTH)
|
||||
done_output: str | None = Field(None, max_length=MAX_STRING_LENGTH)
|
||||
scheduled_task_id: str | None = None
|
||||
started_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
finished_at: datetime | None = None
|
||||
agent_state: dict = Field(default_factory=dict)
|
||||
user_feedback_type: str | None = Field(None, max_length=10) # UserFeedbackType enum value as string
|
||||
user_comment: str | None = Field(None, max_length=MAX_COMMENT_LENGTH)
|
||||
gif_url: str | None = Field(None, max_length=MAX_URL_LENGTH)
|
||||
|
||||
@classmethod
|
||||
def from_agent(cls, agent) -> 'CreateAgentTaskEvent':
|
||||
"""Create a CreateAgentTaskEvent from an Agent instance"""
|
||||
return cls(
|
||||
id=str(agent.task_id),
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
agent_session_id=str(agent.session_id),
|
||||
task=agent.task,
|
||||
llm_model=agent.llm.model_name,
|
||||
agent_state=agent.state.model_dump() if hasattr(agent.state, 'model_dump') else {},
|
||||
stopped=False,
|
||||
paused=False,
|
||||
done_output=None,
|
||||
started_at=datetime.fromtimestamp(agent._task_start_time, tz=timezone.utc),
|
||||
finished_at=None,
|
||||
user_feedback_type=None,
|
||||
user_comment=None,
|
||||
gif_url=None,
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentSessionEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255)
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
browser_session_id: str = Field(max_length=255)
|
||||
browser_session_live_url: str = Field(max_length=MAX_URL_LENGTH)
|
||||
browser_session_cdp_url: str = Field(max_length=MAX_URL_LENGTH)
|
||||
browser_session_stopped: bool = False
|
||||
browser_session_stopped_at: datetime | None = None
|
||||
is_source_api: bool | None = None
|
||||
browser_state: dict = Field(default_factory=dict)
|
||||
browser_session_data: dict | None = None
|
||||
|
||||
@classmethod
|
||||
def from_agent(cls, agent) -> 'CreateAgentSessionEvent':
|
||||
"""Create a CreateAgentSessionEvent from an Agent instance"""
|
||||
return cls(
|
||||
id=str(agent.session_id),
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
browser_session_id=agent.browser_session.id,
|
||||
browser_session_live_url='', # To be filled by cloud handler
|
||||
browser_session_cdp_url='', # To be filled by cloud handler
|
||||
browser_state={
|
||||
'viewport': agent.browser_profile.viewport if agent.browser_profile else {'width': 1280, 'height': 720},
|
||||
'user_agent': agent.browser_profile.user_agent if agent.browser_profile else None,
|
||||
'headless': agent.browser_profile.headless if agent.browser_profile else True,
|
||||
'initial_url': None, # Will be updated during execution
|
||||
'final_url': None, # Will be updated during execution
|
||||
'total_pages_visited': 0, # Will be updated during execution
|
||||
'session_duration_seconds': 0, # Will be updated during execution
|
||||
},
|
||||
browser_session_data={
|
||||
'cookies': [],
|
||||
'secrets': {},
|
||||
# TODO: send secrets safely so tasks can be replayed on cloud seamlessly
|
||||
# 'secrets': dict(agent.sensitive_data) if agent.sensitive_data else {},
|
||||
'allowed_domains': agent.browser_profile.allowed_domains if agent.browser_profile else [],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class UpdateAgentSessionEvent(BaseEvent):
|
||||
"""Event to update an existing agent session"""
|
||||
|
||||
# Model fields
|
||||
id: str # Session ID to update
|
||||
user_id: str = Field(max_length=255)
|
||||
device_id: str | None = Field(None, max_length=255)
|
||||
browser_session_stopped: bool | None = None
|
||||
browser_session_stopped_at: datetime | None = None
|
||||
end_reason: str | None = Field(None, max_length=100) # Why the session ended
|
||||
@@ -0,0 +1,424 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from browser_use.agent.views import AgentHistoryList
|
||||
from browser_use.browser.views import PLACEHOLDER_4PX_SCREENSHOT
|
||||
from browser_use.config import CONFIG
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from PIL import Image, ImageFont
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def decode_unicode_escapes_to_utf8(text: str) -> str:
|
||||
"""Handle decoding any unicode escape sequences embedded in a string (needed to render non-ASCII languages like chinese or arabic in the GIF overlay text)"""
|
||||
|
||||
if r'\u' not in text:
|
||||
# doesn't have any escape sequences that need to be decoded
|
||||
return text
|
||||
|
||||
try:
|
||||
# Try to decode Unicode escape sequences
|
||||
return text.encode('latin1').decode('unicode_escape')
|
||||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||||
# logger.debug(f"Failed to decode unicode escape sequences while generating gif text: {text}")
|
||||
return text
|
||||
|
||||
|
||||
def create_history_gif(
|
||||
task: str,
|
||||
history: AgentHistoryList,
|
||||
#
|
||||
output_path: str = 'agent_history.gif',
|
||||
duration: int = 3000,
|
||||
show_goals: bool = True,
|
||||
show_task: bool = True,
|
||||
show_logo: bool = False,
|
||||
font_size: int = 40,
|
||||
title_font_size: int = 56,
|
||||
goal_font_size: int = 44,
|
||||
margin: int = 40,
|
||||
line_spacing: float = 1.5,
|
||||
) -> None:
|
||||
"""Create a GIF from the agent's history with overlaid task and goal text."""
|
||||
if not history.history:
|
||||
logger.warning('No history to create GIF from')
|
||||
return
|
||||
|
||||
from PIL import Image, ImageFont
|
||||
|
||||
images = []
|
||||
|
||||
# if history is empty, we can't create a gif
|
||||
if not history.history:
|
||||
logger.warning('No history to create GIF from')
|
||||
return
|
||||
|
||||
# Get all screenshots from history (including None placeholders)
|
||||
screenshots = history.screenshots(return_none_if_not_screenshot=True)
|
||||
|
||||
if not screenshots:
|
||||
logger.warning('No screenshots found in history')
|
||||
return
|
||||
|
||||
# Find the first non-placeholder screenshot
|
||||
# A screenshot is considered a placeholder if:
|
||||
# 1. It's the exact 4px placeholder for about:blank pages, OR
|
||||
# 2. It comes from a new tab page (chrome://newtab/, about:blank, etc.)
|
||||
first_real_screenshot = None
|
||||
for screenshot in screenshots:
|
||||
if screenshot and screenshot != PLACEHOLDER_4PX_SCREENSHOT:
|
||||
first_real_screenshot = screenshot
|
||||
break
|
||||
|
||||
if not first_real_screenshot:
|
||||
logger.warning('No valid screenshots found (all are placeholders or from new tab pages)')
|
||||
return
|
||||
|
||||
# Try to load nicer fonts
|
||||
try:
|
||||
# Try different font options in order of preference
|
||||
# ArialUni is a font that comes with Office and can render most non-alphabet characters
|
||||
font_options = [
|
||||
'PingFang',
|
||||
'STHeiti Medium',
|
||||
'Microsoft YaHei', # 微软雅黑
|
||||
'SimHei', # 黑体
|
||||
'SimSun', # 宋体
|
||||
'Noto Sans CJK SC', # 思源黑体
|
||||
'WenQuanYi Micro Hei', # 文泉驿微米黑
|
||||
'Helvetica',
|
||||
'Arial',
|
||||
'DejaVuSans',
|
||||
'Verdana',
|
||||
]
|
||||
font_loaded = False
|
||||
|
||||
for font_name in font_options:
|
||||
try:
|
||||
if platform.system() == 'Windows':
|
||||
# Need to specify the abs font path on Windows
|
||||
font_name = os.path.join(CONFIG.WIN_FONT_DIR, font_name + '.ttf')
|
||||
regular_font = ImageFont.truetype(font_name, font_size)
|
||||
title_font = ImageFont.truetype(font_name, title_font_size)
|
||||
goal_font = ImageFont.truetype(font_name, goal_font_size)
|
||||
font_loaded = True
|
||||
break
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
if not font_loaded:
|
||||
raise OSError('No preferred fonts found')
|
||||
|
||||
except OSError:
|
||||
regular_font = ImageFont.load_default()
|
||||
title_font = ImageFont.load_default()
|
||||
|
||||
goal_font = regular_font
|
||||
|
||||
# Load logo if requested
|
||||
logo = None
|
||||
if show_logo:
|
||||
try:
|
||||
logo = Image.open('./static/browser-use.png')
|
||||
# Resize logo to be small (e.g., 40px height)
|
||||
logo_height = 150
|
||||
aspect_ratio = logo.width / logo.height
|
||||
logo_width = int(logo_height * aspect_ratio)
|
||||
logo = logo.resize((logo_width, logo_height), Image.Resampling.LANCZOS)
|
||||
except Exception as e:
|
||||
logger.warning(f'Could not load logo: {e}')
|
||||
|
||||
# Create task frame if requested
|
||||
if show_task and task:
|
||||
# Find the first non-placeholder screenshot for the task frame
|
||||
first_real_screenshot = None
|
||||
for item in history.history:
|
||||
screenshot_b64 = item.state.get_screenshot()
|
||||
if screenshot_b64 and screenshot_b64 != PLACEHOLDER_4PX_SCREENSHOT:
|
||||
first_real_screenshot = screenshot_b64
|
||||
break
|
||||
|
||||
if first_real_screenshot:
|
||||
task_frame = _create_task_frame(
|
||||
task,
|
||||
first_real_screenshot,
|
||||
title_font, # type: ignore
|
||||
regular_font, # type: ignore
|
||||
logo,
|
||||
line_spacing,
|
||||
)
|
||||
images.append(task_frame)
|
||||
else:
|
||||
logger.warning('No real screenshots found for task frame, skipping task frame')
|
||||
|
||||
# Process each history item with its corresponding screenshot
|
||||
for i, (item, screenshot) in enumerate(zip(history.history, screenshots), 1):
|
||||
if not screenshot:
|
||||
continue
|
||||
|
||||
# Skip placeholder screenshots from about:blank pages
|
||||
# These are 4x4 white PNGs encoded as a specific base64 string
|
||||
if screenshot == PLACEHOLDER_4PX_SCREENSHOT:
|
||||
logger.debug(f'Skipping placeholder screenshot from about:blank page at step {i}')
|
||||
continue
|
||||
|
||||
# Skip screenshots from new tab pages
|
||||
from browser_use.utils import is_new_tab_page
|
||||
|
||||
if is_new_tab_page(item.state.url):
|
||||
logger.debug(f'Skipping screenshot from new tab page ({item.state.url}) at step {i}')
|
||||
continue
|
||||
|
||||
# Convert base64 screenshot to PIL Image
|
||||
img_data = base64.b64decode(screenshot)
|
||||
image = Image.open(io.BytesIO(img_data))
|
||||
|
||||
if show_goals and item.model_output:
|
||||
image = _add_overlay_to_image(
|
||||
image=image,
|
||||
step_number=i,
|
||||
goal_text=item.model_output.current_state.next_goal,
|
||||
regular_font=regular_font, # type: ignore
|
||||
title_font=title_font, # type: ignore
|
||||
margin=margin,
|
||||
logo=logo,
|
||||
)
|
||||
|
||||
images.append(image)
|
||||
|
||||
if images:
|
||||
# Save the GIF
|
||||
images[0].save(
|
||||
output_path,
|
||||
save_all=True,
|
||||
append_images=images[1:],
|
||||
duration=duration,
|
||||
loop=0,
|
||||
optimize=False,
|
||||
)
|
||||
logger.info(f'Created GIF at {output_path}')
|
||||
else:
|
||||
logger.warning('No images found in history to create GIF')
|
||||
|
||||
|
||||
def _create_task_frame(
|
||||
task: str,
|
||||
first_screenshot: str,
|
||||
title_font: ImageFont.FreeTypeFont,
|
||||
regular_font: ImageFont.FreeTypeFont,
|
||||
logo: Image.Image | None = None,
|
||||
line_spacing: float = 1.5,
|
||||
) -> Image.Image:
|
||||
"""Create initial frame showing the task."""
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
img_data = base64.b64decode(first_screenshot)
|
||||
template = Image.open(io.BytesIO(img_data))
|
||||
image = Image.new('RGB', template.size, (0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
|
||||
# Calculate vertical center of image
|
||||
center_y = image.height // 2
|
||||
|
||||
# Draw task text with dynamic font size based on task length
|
||||
margin = 140 # Increased margin
|
||||
max_width = image.width - (2 * margin)
|
||||
|
||||
# Dynamic font size calculation based on task length
|
||||
# Start with base font size (regular + 16)
|
||||
base_font_size = regular_font.size + 16
|
||||
min_font_size = max(regular_font.size - 10, 16) # Don't go below 16pt
|
||||
max_font_size = base_font_size # Cap at the base font size
|
||||
|
||||
# Calculate dynamic font size based on text length and complexity
|
||||
# Longer texts get progressively smaller fonts
|
||||
text_length = len(task)
|
||||
if text_length > 200:
|
||||
# For very long text, reduce font size logarithmically
|
||||
font_size = max(base_font_size - int(10 * (text_length / 200)), min_font_size)
|
||||
else:
|
||||
font_size = base_font_size
|
||||
|
||||
# Try to create a larger font, but fall back to regular font if it fails
|
||||
try:
|
||||
larger_font = ImageFont.truetype(regular_font.path, font_size) # type: ignore
|
||||
except (OSError, AttributeError):
|
||||
# Fall back to regular font if .path is not available or font loading fails
|
||||
larger_font = regular_font
|
||||
|
||||
# Generate wrapped text with the calculated font size
|
||||
wrapped_text = _wrap_text(task, larger_font, max_width)
|
||||
|
||||
# Calculate line height with spacing
|
||||
line_height = larger_font.size * line_spacing
|
||||
|
||||
# Split text into lines and draw with custom spacing
|
||||
lines = wrapped_text.split('\n')
|
||||
total_height = line_height * len(lines)
|
||||
|
||||
# Start position for first line
|
||||
text_y = center_y - (total_height / 2) + 50 # Shifted down slightly
|
||||
|
||||
for line in lines:
|
||||
# Get line width for centering
|
||||
line_bbox = draw.textbbox((0, 0), line, font=larger_font)
|
||||
text_x = (image.width - (line_bbox[2] - line_bbox[0])) // 2
|
||||
|
||||
draw.text(
|
||||
(text_x, text_y),
|
||||
line,
|
||||
font=larger_font,
|
||||
fill=(255, 255, 255),
|
||||
)
|
||||
text_y += line_height
|
||||
|
||||
# Add logo if provided (top right corner)
|
||||
if logo:
|
||||
logo_margin = 20
|
||||
logo_x = image.width - logo.width - logo_margin
|
||||
image.paste(logo, (logo_x, logo_margin), logo if logo.mode == 'RGBA' else None)
|
||||
|
||||
return image
|
||||
|
||||
|
||||
def _add_overlay_to_image(
|
||||
image: Image.Image,
|
||||
step_number: int,
|
||||
goal_text: str,
|
||||
regular_font: ImageFont.FreeTypeFont,
|
||||
title_font: ImageFont.FreeTypeFont,
|
||||
margin: int,
|
||||
logo: Image.Image | None = None,
|
||||
display_step: bool = True,
|
||||
text_color: tuple[int, int, int, int] = (255, 255, 255, 255),
|
||||
text_box_color: tuple[int, int, int, int] = (0, 0, 0, 255),
|
||||
) -> Image.Image:
|
||||
"""Add step number and goal overlay to an image."""
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
goal_text = decode_unicode_escapes_to_utf8(goal_text)
|
||||
image = image.convert('RGBA')
|
||||
txt_layer = Image.new('RGBA', image.size, (0, 0, 0, 0))
|
||||
draw = ImageDraw.Draw(txt_layer)
|
||||
if display_step:
|
||||
# Add step number (bottom left)
|
||||
step_text = str(step_number)
|
||||
step_bbox = draw.textbbox((0, 0), step_text, font=title_font)
|
||||
step_width = step_bbox[2] - step_bbox[0]
|
||||
step_height = step_bbox[3] - step_bbox[1]
|
||||
|
||||
# Position step number in bottom left
|
||||
x_step = margin + 10 # Slight additional offset from edge
|
||||
y_step = image.height - margin - step_height - 10 # Slight offset from bottom
|
||||
|
||||
# Draw rounded rectangle background for step number
|
||||
padding = 20 # Increased padding
|
||||
step_bg_bbox = (
|
||||
x_step - padding,
|
||||
y_step - padding,
|
||||
x_step + step_width + padding,
|
||||
y_step + step_height + padding,
|
||||
)
|
||||
draw.rounded_rectangle(
|
||||
step_bg_bbox,
|
||||
radius=15, # Add rounded corners
|
||||
fill=text_box_color,
|
||||
)
|
||||
|
||||
# Draw step number
|
||||
draw.text(
|
||||
(x_step, y_step),
|
||||
step_text,
|
||||
font=title_font,
|
||||
fill=text_color,
|
||||
)
|
||||
|
||||
# Draw goal text (centered, bottom)
|
||||
max_width = image.width - (4 * margin)
|
||||
wrapped_goal = _wrap_text(goal_text, title_font, max_width)
|
||||
goal_bbox = draw.multiline_textbbox((0, 0), wrapped_goal, font=title_font)
|
||||
goal_width = goal_bbox[2] - goal_bbox[0]
|
||||
goal_height = goal_bbox[3] - goal_bbox[1]
|
||||
|
||||
# Center goal text horizontally, place above step number
|
||||
x_goal = (image.width - goal_width) // 2
|
||||
y_goal = y_step - goal_height - padding * 4 # More space between step and goal
|
||||
|
||||
# Draw rounded rectangle background for goal
|
||||
padding_goal = 25 # Increased padding for goal
|
||||
goal_bg_bbox = (
|
||||
x_goal - padding_goal, # Remove extra space for logo
|
||||
y_goal - padding_goal,
|
||||
x_goal + goal_width + padding_goal,
|
||||
y_goal + goal_height + padding_goal,
|
||||
)
|
||||
draw.rounded_rectangle(
|
||||
goal_bg_bbox,
|
||||
radius=15, # Add rounded corners
|
||||
fill=text_box_color,
|
||||
)
|
||||
|
||||
# Draw goal text
|
||||
draw.multiline_text(
|
||||
(x_goal, y_goal),
|
||||
wrapped_goal,
|
||||
font=title_font,
|
||||
fill=text_color,
|
||||
align='center',
|
||||
)
|
||||
|
||||
# Add logo if provided (top right corner)
|
||||
if logo:
|
||||
logo_layer = Image.new('RGBA', image.size, (0, 0, 0, 0))
|
||||
logo_margin = 20
|
||||
logo_x = image.width - logo.width - logo_margin
|
||||
logo_layer.paste(logo, (logo_x, logo_margin), logo if logo.mode == 'RGBA' else None)
|
||||
txt_layer = Image.alpha_composite(logo_layer, txt_layer)
|
||||
|
||||
# Composite and convert
|
||||
result = Image.alpha_composite(image, txt_layer)
|
||||
return result.convert('RGB')
|
||||
|
||||
|
||||
def _wrap_text(text: str, font: ImageFont.FreeTypeFont, max_width: int) -> str:
|
||||
"""
|
||||
Wrap text to fit within a given width.
|
||||
|
||||
Args:
|
||||
text: Text to wrap
|
||||
font: Font to use for text
|
||||
max_width: Maximum width in pixels
|
||||
|
||||
Returns:
|
||||
Wrapped text with newlines
|
||||
"""
|
||||
text = decode_unicode_escapes_to_utf8(text)
|
||||
words = text.split()
|
||||
lines = []
|
||||
current_line = []
|
||||
|
||||
for word in words:
|
||||
current_line.append(word)
|
||||
line = ' '.join(current_line)
|
||||
bbox = font.getbbox(line)
|
||||
if bbox[2] > max_width:
|
||||
if len(current_line) == 1:
|
||||
lines.append(current_line.pop())
|
||||
else:
|
||||
current_line.pop()
|
||||
lines.append(' '.join(current_line))
|
||||
current_line = [word]
|
||||
|
||||
if current_line:
|
||||
lines.append(' '.join(current_line))
|
||||
|
||||
return '\n'.join(lines)
|
||||
@@ -0,0 +1,422 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Literal
|
||||
|
||||
from browser_use.agent.message_manager.views import (
|
||||
HistoryItem,
|
||||
)
|
||||
from browser_use.agent.prompts import AgentMessagePrompt
|
||||
from browser_use.agent.views import (
|
||||
ActionResult,
|
||||
AgentOutput,
|
||||
AgentStepInfo,
|
||||
MessageManagerState,
|
||||
)
|
||||
from browser_use.browser.views import BrowserStateSummary
|
||||
from browser_use.filesystem.file_system import FileSystem
|
||||
from browser_use.llm.messages import (
|
||||
BaseMessage,
|
||||
ContentPartImageParam,
|
||||
ContentPartTextParam,
|
||||
SystemMessage,
|
||||
)
|
||||
from browser_use.observability import observe_debug
|
||||
from browser_use.utils import match_url_with_domain_pattern, time_execution_sync
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ========== Logging Helper Functions ==========
|
||||
# These functions are used ONLY for formatting debug log output.
|
||||
# They do NOT affect the actual message content sent to the LLM.
|
||||
# All logging functions start with _log_ for easy identification.
|
||||
|
||||
|
||||
def _log_get_message_emoji(message: BaseMessage) -> str:
|
||||
"""Get emoji for a message type - used only for logging display"""
|
||||
emoji_map = {
|
||||
'UserMessage': '💬',
|
||||
'SystemMessage': '🧠',
|
||||
'AssistantMessage': '🔨',
|
||||
}
|
||||
return emoji_map.get(message.__class__.__name__, '🎮')
|
||||
|
||||
|
||||
def _log_format_message_line(message: BaseMessage, content: str, is_last_message: bool, terminal_width: int) -> list[str]:
|
||||
"""Format a single message for logging display"""
|
||||
try:
|
||||
lines = []
|
||||
|
||||
# Get emoji and token info
|
||||
emoji = _log_get_message_emoji(message)
|
||||
# token_str = str(message.metadata.tokens).rjust(4)
|
||||
# TODO: fix the token count
|
||||
token_str = '??? (TODO)'
|
||||
prefix = f'{emoji}[{token_str}]: '
|
||||
|
||||
# Calculate available width (emoji=2 visual cols + [token]: =8 chars)
|
||||
content_width = terminal_width - 10
|
||||
|
||||
# Handle last message wrapping
|
||||
if is_last_message and len(content) > content_width:
|
||||
# Find a good break point
|
||||
break_point = content.rfind(' ', 0, content_width)
|
||||
if break_point > content_width * 0.7: # Keep at least 70% of line
|
||||
first_line = content[:break_point]
|
||||
rest = content[break_point + 1 :]
|
||||
else:
|
||||
# No good break point, just truncate
|
||||
first_line = content[:content_width]
|
||||
rest = content[content_width:]
|
||||
|
||||
lines.append(prefix + first_line)
|
||||
|
||||
# Second line with 10-space indent
|
||||
if rest:
|
||||
if len(rest) > terminal_width - 10:
|
||||
rest = rest[: terminal_width - 10]
|
||||
lines.append(' ' * 10 + rest)
|
||||
else:
|
||||
# Single line - truncate if needed
|
||||
if len(content) > content_width:
|
||||
content = content[:content_width]
|
||||
lines.append(prefix + content)
|
||||
|
||||
return lines
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to format message line for logging: {e}')
|
||||
# Return a simple fallback line
|
||||
return ['❓[ ?]: [Error formatting message]']
|
||||
|
||||
|
||||
# ========== End of Logging Helper Functions ==========
|
||||
|
||||
|
||||
class MessageManager:
|
||||
vision_detail_level: Literal['auto', 'low', 'high']
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
task: str,
|
||||
system_message: SystemMessage,
|
||||
file_system: FileSystem,
|
||||
state: MessageManagerState = MessageManagerState(),
|
||||
use_thinking: bool = True,
|
||||
include_attributes: list[str] | None = None,
|
||||
sensitive_data: dict[str, str | dict[str, str]] | None = None,
|
||||
max_history_items: int | None = None,
|
||||
vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
|
||||
include_tool_call_examples: bool = False,
|
||||
include_recent_events: bool = False,
|
||||
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
|
||||
):
|
||||
self.task = task
|
||||
self.state = state
|
||||
self.system_prompt = system_message
|
||||
self.file_system = file_system
|
||||
self.sensitive_data_description = ''
|
||||
self.use_thinking = use_thinking
|
||||
self.max_history_items = max_history_items
|
||||
self.vision_detail_level = vision_detail_level
|
||||
self.include_tool_call_examples = include_tool_call_examples
|
||||
self.include_recent_events = include_recent_events
|
||||
self.sample_images = sample_images
|
||||
|
||||
assert max_history_items is None or max_history_items > 5, 'max_history_items must be None or greater than 5'
|
||||
|
||||
# Store settings as direct attributes instead of in a settings object
|
||||
self.include_attributes = include_attributes or []
|
||||
self.sensitive_data = sensitive_data
|
||||
self.last_input_messages = []
|
||||
# Only initialize messages if state is empty
|
||||
if len(self.state.history.get_messages()) == 0:
|
||||
self._set_message_with_type(self.system_prompt, 'system')
|
||||
|
||||
@property
|
||||
def agent_history_description(self) -> str:
|
||||
"""Build agent history description from list of items, respecting max_history_items limit"""
|
||||
if self.max_history_items is None:
|
||||
# Include all items
|
||||
return '\n'.join(item.to_string() for item in self.state.agent_history_items)
|
||||
|
||||
total_items = len(self.state.agent_history_items)
|
||||
|
||||
# If we have fewer items than the limit, just return all items
|
||||
if total_items <= self.max_history_items:
|
||||
return '\n'.join(item.to_string() for item in self.state.agent_history_items)
|
||||
|
||||
# We have more items than the limit, so we need to omit some
|
||||
omitted_count = total_items - self.max_history_items
|
||||
|
||||
# Show first item + omitted message + most recent (max_history_items - 1) items
|
||||
# The omitted message doesn't count against the limit, only real history items do
|
||||
recent_items_count = self.max_history_items - 1 # -1 for first item
|
||||
|
||||
items_to_include = [
|
||||
self.state.agent_history_items[0].to_string(), # Keep first item (initialization)
|
||||
f'<sys>[... {omitted_count} previous steps omitted...]</sys>',
|
||||
]
|
||||
# Add most recent items
|
||||
items_to_include.extend([item.to_string() for item in self.state.agent_history_items[-recent_items_count:]])
|
||||
|
||||
return '\n'.join(items_to_include)
|
||||
|
||||
def add_new_task(self, new_task: str) -> None:
|
||||
new_task = '<follow_up_user_request> ' + new_task.strip() + ' </follow_up_user_request>'
|
||||
if '<initial_user_request>' not in self.task:
|
||||
self.task = '<initial_user_request>' + self.task + '</initial_user_request>'
|
||||
self.task += '\n' + new_task
|
||||
task_update_item = HistoryItem(system_message=new_task)
|
||||
self.state.agent_history_items.append(task_update_item)
|
||||
|
||||
def _update_agent_history_description(
|
||||
self,
|
||||
model_output: AgentOutput | None = None,
|
||||
result: list[ActionResult] | None = None,
|
||||
step_info: AgentStepInfo | None = None,
|
||||
) -> None:
|
||||
"""Update the agent history description"""
|
||||
|
||||
if result is None:
|
||||
result = []
|
||||
step_number = step_info.step_number if step_info else None
|
||||
|
||||
self.state.read_state_description = ''
|
||||
|
||||
action_results = ''
|
||||
result_len = len(result)
|
||||
read_state_idx = 0
|
||||
for idx, action_result in enumerate(result):
|
||||
if action_result.include_extracted_content_only_once and action_result.extracted_content:
|
||||
self.state.read_state_description += (
|
||||
f'<read_state_{read_state_idx}>\n{action_result.extracted_content}\n</read_state_{read_state_idx}>\n'
|
||||
)
|
||||
read_state_idx += 1
|
||||
logger.debug(f'Added extracted_content to read_state_description: {action_result.extracted_content}')
|
||||
|
||||
if action_result.long_term_memory:
|
||||
action_results += f'{action_result.long_term_memory}\n'
|
||||
logger.debug(f'Added long_term_memory to action_results: {action_result.long_term_memory}')
|
||||
elif action_result.extracted_content and not action_result.include_extracted_content_only_once:
|
||||
action_results += f'{action_result.extracted_content}\n'
|
||||
logger.debug(f'Added extracted_content to action_results: {action_result.extracted_content}')
|
||||
|
||||
if action_result.error:
|
||||
if len(action_result.error) > 200:
|
||||
error_text = action_result.error[:100] + '......' + action_result.error[-100:]
|
||||
else:
|
||||
error_text = action_result.error
|
||||
action_results += f'{error_text}\n'
|
||||
logger.debug(f'Added error to action_results: {error_text}')
|
||||
|
||||
self.state.read_state_description = self.state.read_state_description.strip('\n')
|
||||
|
||||
if action_results:
|
||||
action_results = f'Result:\n{action_results}'
|
||||
action_results = action_results.strip('\n') if action_results else None
|
||||
|
||||
# Build the history item
|
||||
if model_output is None:
|
||||
# Add history item for initial actions (step 0) or errors (step > 0)
|
||||
if step_number is not None:
|
||||
if step_number == 0 and action_results:
|
||||
# Step 0 with initial action results
|
||||
history_item = HistoryItem(step_number=step_number, action_results=action_results)
|
||||
self.state.agent_history_items.append(history_item)
|
||||
elif step_number > 0:
|
||||
# Error case for steps > 0
|
||||
history_item = HistoryItem(step_number=step_number, error='Agent failed to output in the right format.')
|
||||
self.state.agent_history_items.append(history_item)
|
||||
else:
|
||||
history_item = HistoryItem(
|
||||
step_number=step_number,
|
||||
evaluation_previous_goal=model_output.current_state.evaluation_previous_goal,
|
||||
memory=model_output.current_state.memory,
|
||||
next_goal=model_output.current_state.next_goal,
|
||||
action_results=action_results,
|
||||
)
|
||||
self.state.agent_history_items.append(history_item)
|
||||
|
||||
def _get_sensitive_data_description(self, current_page_url) -> str:
|
||||
sensitive_data = self.sensitive_data
|
||||
if not sensitive_data:
|
||||
return ''
|
||||
|
||||
# Collect placeholders for sensitive data
|
||||
placeholders: set[str] = set()
|
||||
|
||||
for key, value in sensitive_data.items():
|
||||
if isinstance(value, dict):
|
||||
# New format: {domain: {key: value}}
|
||||
if current_page_url and match_url_with_domain_pattern(current_page_url, key, True):
|
||||
placeholders.update(value.keys())
|
||||
else:
|
||||
# Old format: {key: value}
|
||||
placeholders.add(key)
|
||||
|
||||
if placeholders:
|
||||
placeholder_list = sorted(list(placeholders))
|
||||
info = f'Here are placeholders for sensitive data:\n{placeholder_list}\n'
|
||||
info += 'To use them, write <secret>the placeholder name</secret>'
|
||||
return info
|
||||
|
||||
return ''
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='create_state_messages')
|
||||
@time_execution_sync('--create_state_messages')
|
||||
def create_state_messages(
|
||||
self,
|
||||
browser_state_summary: BrowserStateSummary,
|
||||
model_output: AgentOutput | None = None,
|
||||
result: list[ActionResult] | None = None,
|
||||
step_info: AgentStepInfo | None = None,
|
||||
use_vision=True,
|
||||
page_filtered_actions: str | None = None,
|
||||
sensitive_data=None,
|
||||
available_file_paths: list[str] | None = None, # Always pass current available_file_paths
|
||||
) -> None:
|
||||
"""Create single state message with all content"""
|
||||
|
||||
# Clear contextual messages from previous steps to prevent accumulation
|
||||
self.state.history.context_messages.clear()
|
||||
|
||||
# First, update the agent history items with the latest step results
|
||||
self._update_agent_history_description(model_output, result, step_info)
|
||||
|
||||
# Use the passed sensitive_data parameter, falling back to instance variable
|
||||
effective_sensitive_data = sensitive_data if sensitive_data is not None else self.sensitive_data
|
||||
if effective_sensitive_data is not None:
|
||||
# Update instance variable to keep it in sync
|
||||
self.sensitive_data = effective_sensitive_data
|
||||
self.sensitive_data_description = self._get_sensitive_data_description(browser_state_summary.url)
|
||||
|
||||
# Use only the current screenshot
|
||||
screenshots = []
|
||||
if browser_state_summary.screenshot:
|
||||
screenshots.append(browser_state_summary.screenshot)
|
||||
|
||||
# Create single state message with all content
|
||||
assert browser_state_summary
|
||||
state_message = AgentMessagePrompt(
|
||||
browser_state_summary=browser_state_summary,
|
||||
file_system=self.file_system,
|
||||
agent_history_description=self.agent_history_description,
|
||||
read_state_description=self.state.read_state_description,
|
||||
task=self.task,
|
||||
include_attributes=self.include_attributes,
|
||||
step_info=step_info,
|
||||
page_filtered_actions=page_filtered_actions,
|
||||
sensitive_data=self.sensitive_data_description,
|
||||
available_file_paths=available_file_paths,
|
||||
screenshots=screenshots,
|
||||
vision_detail_level=self.vision_detail_level,
|
||||
include_recent_events=self.include_recent_events,
|
||||
sample_images=self.sample_images,
|
||||
).get_user_message(use_vision)
|
||||
|
||||
# Set the state message with caching enabled
|
||||
self._set_message_with_type(state_message, 'state')
|
||||
|
||||
def _log_history_lines(self) -> str:
|
||||
"""Generate a formatted log string of message history for debugging / printing to terminal"""
|
||||
# TODO: fix logging
|
||||
|
||||
# try:
|
||||
# total_input_tokens = 0
|
||||
# message_lines = []
|
||||
# terminal_width = shutil.get_terminal_size((80, 20)).columns
|
||||
|
||||
# for i, m in enumerate(self.state.history.messages):
|
||||
# try:
|
||||
# total_input_tokens += m.metadata.tokens
|
||||
# is_last_message = i == len(self.state.history.messages) - 1
|
||||
|
||||
# # Extract content for logging
|
||||
# content = _log_extract_message_content(m.message, is_last_message, m.metadata)
|
||||
|
||||
# # Format the message line(s)
|
||||
# lines = _log_format_message_line(m, content, is_last_message, terminal_width)
|
||||
# message_lines.extend(lines)
|
||||
# except Exception as e:
|
||||
# logger.warning(f'Failed to format message {i} for logging: {e}')
|
||||
# # Add a fallback line for this message
|
||||
# message_lines.append('❓[ ?]: [Error formatting this message]')
|
||||
|
||||
# # Build final log message
|
||||
# return (
|
||||
# f'📜 LLM Message history ({len(self.state.history.messages)} messages, {total_input_tokens} tokens):\n'
|
||||
# + '\n'.join(message_lines)
|
||||
# )
|
||||
# except Exception as e:
|
||||
# logger.warning(f'Failed to generate history log: {e}')
|
||||
# # Return a minimal fallback message
|
||||
# return f'📜 LLM Message history (error generating log: {e})'
|
||||
|
||||
return ''
|
||||
|
||||
@time_execution_sync('--get_messages')
|
||||
def get_messages(self) -> list[BaseMessage]:
|
||||
"""Get current message list, potentially trimmed to max tokens"""
|
||||
|
||||
# Log message history for debugging
|
||||
logger.debug(self._log_history_lines())
|
||||
self.last_input_messages = self.state.history.get_messages()
|
||||
return self.last_input_messages
|
||||
|
||||
def _set_message_with_type(self, message: BaseMessage, message_type: Literal['system', 'state']) -> None:
|
||||
"""Replace a specific state message slot with a new message"""
|
||||
# Don't filter system and state messages - they should contain placeholder tags or normal conversation
|
||||
if message_type == 'system':
|
||||
self.state.history.system_message = message
|
||||
elif message_type == 'state':
|
||||
self.state.history.state_message = message
|
||||
else:
|
||||
raise ValueError(f'Invalid state message type: {message_type}')
|
||||
|
||||
def _add_context_message(self, message: BaseMessage) -> None:
|
||||
"""Add a contextual message specific to this step (e.g., validation errors, retry instructions, timeout warnings)"""
|
||||
# Don't filter context messages - they should contain normal conversation or error messages
|
||||
self.state.history.context_messages.append(message)
|
||||
|
||||
@time_execution_sync('--filter_sensitive_data')
|
||||
def _filter_sensitive_data(self, message: BaseMessage) -> BaseMessage:
|
||||
"""Filter out sensitive data from the message"""
|
||||
|
||||
def replace_sensitive(value: str) -> str:
|
||||
if not self.sensitive_data:
|
||||
return value
|
||||
|
||||
# Collect all sensitive values, immediately converting old format to new format
|
||||
sensitive_values: dict[str, str] = {}
|
||||
|
||||
# Process all sensitive data entries
|
||||
for key_or_domain, content in self.sensitive_data.items():
|
||||
if isinstance(content, dict):
|
||||
# Already in new format: {domain: {key: value}}
|
||||
for key, val in content.items():
|
||||
if val: # Skip empty values
|
||||
sensitive_values[key] = val
|
||||
elif content: # Old format: {key: value} - convert to new format internally
|
||||
# We treat this as if it was {'http*://*': {key_or_domain: content}}
|
||||
sensitive_values[key_or_domain] = content
|
||||
|
||||
# If there are no valid sensitive data entries, just return the original value
|
||||
if not sensitive_values:
|
||||
logger.warning('No valid entries found in sensitive_data dictionary')
|
||||
return value
|
||||
|
||||
# Replace all valid sensitive data values with their placeholder tags
|
||||
for key, val in sensitive_values.items():
|
||||
value = value.replace(val, f'<secret>{key}</secret>')
|
||||
|
||||
return value
|
||||
|
||||
if isinstance(message.content, str):
|
||||
message.content = replace_sensitive(message.content)
|
||||
elif isinstance(message.content, list):
|
||||
for i, item in enumerate(message.content):
|
||||
if isinstance(item, ContentPartTextParam):
|
||||
item.text = replace_sensitive(item.text)
|
||||
message.content[i] = item
|
||||
return message
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import anyio
|
||||
|
||||
from browser_use.llm.messages import BaseMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def save_conversation(
|
||||
input_messages: list[BaseMessage],
|
||||
response: Any,
|
||||
target: str | Path,
|
||||
encoding: str | None = None,
|
||||
) -> None:
|
||||
"""Save conversation history to file asynchronously."""
|
||||
target_path = Path(target)
|
||||
# create folders if not exists
|
||||
if target_path.parent:
|
||||
await anyio.Path(target_path.parent).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
await anyio.Path(target_path).write_text(
|
||||
await _format_conversation(input_messages, response),
|
||||
encoding=encoding or 'utf-8',
|
||||
)
|
||||
|
||||
|
||||
async def _format_conversation(messages: list[BaseMessage], response: Any) -> str:
|
||||
"""Format the conversation including messages and response."""
|
||||
lines = []
|
||||
|
||||
# Format messages
|
||||
for message in messages:
|
||||
lines.append(f' {message.role} ')
|
||||
|
||||
lines.append(message.text)
|
||||
lines.append('') # Empty line after each message
|
||||
|
||||
# Format response
|
||||
lines.append(' RESPONSE')
|
||||
lines.append(json.dumps(json.loads(response.model_dump_json(exclude_unset=True)), indent=2))
|
||||
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
# Note: _write_messages_to_file and _write_response_to_file have been merged into _format_conversation
|
||||
# This is more efficient for async operations and reduces file I/O
|
||||
@@ -0,0 +1,98 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from browser_use.llm.messages import (
|
||||
BaseMessage,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class HistoryItem(BaseModel):
|
||||
"""Represents a single agent history item with its data and string representation"""
|
||||
|
||||
step_number: int | None = None
|
||||
evaluation_previous_goal: str | None = None
|
||||
memory: str | None = None
|
||||
next_goal: str | None = None
|
||||
action_results: str | None = None
|
||||
error: str | None = None
|
||||
system_message: str | None = None
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
def model_post_init(self, __context) -> None:
|
||||
"""Validate that error and system_message are not both provided"""
|
||||
if self.error is not None and self.system_message is not None:
|
||||
raise ValueError('Cannot have both error and system_message at the same time')
|
||||
|
||||
def to_string(self) -> str:
|
||||
"""Get string representation of the history item"""
|
||||
step_str = 'step' if self.step_number is not None else 'step_unknown'
|
||||
|
||||
if self.error:
|
||||
return f"""<{step_str}>
|
||||
{self.error}
|
||||
</{step_str}>"""
|
||||
elif self.system_message:
|
||||
return self.system_message
|
||||
else:
|
||||
content_parts = []
|
||||
|
||||
# Only include evaluation_previous_goal if it's not None/empty
|
||||
if self.evaluation_previous_goal:
|
||||
content_parts.append(f'{self.evaluation_previous_goal}')
|
||||
|
||||
# Always include memory
|
||||
if self.memory:
|
||||
content_parts.append(f'{self.memory}')
|
||||
|
||||
# Only include next_goal if it's not None/empty
|
||||
if self.next_goal:
|
||||
content_parts.append(f'{self.next_goal}')
|
||||
|
||||
if self.action_results:
|
||||
content_parts.append(self.action_results)
|
||||
|
||||
content = '\n'.join(content_parts)
|
||||
|
||||
return f"""<{step_str}>
|
||||
{content}
|
||||
</{step_str}>"""
|
||||
|
||||
|
||||
class MessageHistory(BaseModel):
|
||||
"""History of messages"""
|
||||
|
||||
system_message: BaseMessage | None = None
|
||||
state_message: BaseMessage | None = None
|
||||
context_messages: list[BaseMessage] = Field(default_factory=list)
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
def get_messages(self) -> list[BaseMessage]:
|
||||
"""Get all messages in the correct order: system -> state -> contextual"""
|
||||
messages = []
|
||||
if self.system_message:
|
||||
messages.append(self.system_message)
|
||||
if self.state_message:
|
||||
messages.append(self.state_message)
|
||||
messages.extend(self.context_messages)
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
class MessageManagerState(BaseModel):
|
||||
"""Holds the state for MessageManager"""
|
||||
|
||||
history: MessageHistory = Field(default_factory=MessageHistory)
|
||||
tool_id: int = 1
|
||||
agent_history_items: list[HistoryItem] = Field(
|
||||
default_factory=lambda: [HistoryItem(step_number=0, system_message='Agent initialized')]
|
||||
)
|
||||
read_state_description: str = ''
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
@@ -0,0 +1,378 @@
|
||||
import importlib.resources
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Literal, Optional
|
||||
|
||||
from browser_use.dom.views import NodeType, SimplifiedNode
|
||||
from browser_use.llm.messages import ContentPartImageParam, ContentPartTextParam, ImageURL, SystemMessage, UserMessage
|
||||
from browser_use.observability import observe_debug
|
||||
from browser_use.utils import is_new_tab_page
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.agent.views import AgentStepInfo
|
||||
from browser_use.browser.views import BrowserStateSummary
|
||||
from browser_use.filesystem.file_system import FileSystem
|
||||
|
||||
|
||||
class SystemPrompt:
|
||||
def __init__(
|
||||
self,
|
||||
action_description: str,
|
||||
max_actions_per_step: int = 10,
|
||||
override_system_message: str | None = None,
|
||||
extend_system_message: str | None = None,
|
||||
use_thinking: bool = True,
|
||||
flash_mode: bool = False,
|
||||
):
|
||||
self.default_action_description = action_description
|
||||
self.max_actions_per_step = max_actions_per_step
|
||||
self.use_thinking = use_thinking
|
||||
self.flash_mode = flash_mode
|
||||
prompt = ''
|
||||
if override_system_message:
|
||||
prompt = override_system_message
|
||||
else:
|
||||
self._load_prompt_template()
|
||||
prompt = self.prompt_template.format(max_actions=self.max_actions_per_step)
|
||||
|
||||
if extend_system_message:
|
||||
prompt += f'\n{extend_system_message}'
|
||||
|
||||
self.system_message = SystemMessage(content=prompt, cache=True)
|
||||
|
||||
def _load_prompt_template(self) -> None:
|
||||
"""Load the prompt template from the markdown file."""
|
||||
try:
|
||||
# Choose the appropriate template based on flash_mode and use_thinking settings
|
||||
if self.flash_mode:
|
||||
template_filename = 'system_prompt_flash.md'
|
||||
elif self.use_thinking:
|
||||
template_filename = 'system_prompt.md'
|
||||
else:
|
||||
template_filename = 'system_prompt_no_thinking.md'
|
||||
|
||||
# This works both in development and when installed as a package
|
||||
with importlib.resources.files('browser_use.agent').joinpath(template_filename).open('r', encoding='utf-8') as f:
|
||||
self.prompt_template = f.read()
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Failed to load system prompt template: {e}')
|
||||
|
||||
def get_system_message(self) -> SystemMessage:
|
||||
"""
|
||||
Get the system prompt for the agent.
|
||||
|
||||
Returns:
|
||||
SystemMessage: Formatted system prompt
|
||||
"""
|
||||
return self.system_message
|
||||
|
||||
|
||||
# Functions:
|
||||
# {self.default_action_description}
|
||||
|
||||
# Example:
|
||||
# {self.example_response()}
|
||||
# Your AVAILABLE ACTIONS:
|
||||
# {self.default_action_description}
|
||||
|
||||
|
||||
class AgentMessagePrompt:
|
||||
vision_detail_level: Literal['auto', 'low', 'high']
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
browser_state_summary: 'BrowserStateSummary',
|
||||
file_system: 'FileSystem',
|
||||
agent_history_description: str | None = None,
|
||||
read_state_description: str | None = None,
|
||||
task: str | None = None,
|
||||
include_attributes: list[str] | None = None,
|
||||
step_info: Optional['AgentStepInfo'] = None,
|
||||
page_filtered_actions: str | None = None,
|
||||
max_clickable_elements_length: int = 40000,
|
||||
sensitive_data: str | None = None,
|
||||
available_file_paths: list[str] | None = None,
|
||||
screenshots: list[str] | None = None,
|
||||
vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
|
||||
include_recent_events: bool = False,
|
||||
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
|
||||
):
|
||||
self.browser_state: 'BrowserStateSummary' = browser_state_summary
|
||||
self.file_system: 'FileSystem | None' = file_system
|
||||
self.agent_history_description: str | None = agent_history_description
|
||||
self.read_state_description: str | None = read_state_description
|
||||
self.task: str | None = task
|
||||
self.include_attributes = include_attributes
|
||||
self.step_info = step_info
|
||||
self.page_filtered_actions: str | None = page_filtered_actions
|
||||
self.max_clickable_elements_length: int = max_clickable_elements_length
|
||||
self.sensitive_data: str | None = sensitive_data
|
||||
self.available_file_paths: list[str] | None = available_file_paths
|
||||
self.screenshots = screenshots or []
|
||||
self.vision_detail_level = vision_detail_level
|
||||
self.include_recent_events = include_recent_events
|
||||
self.sample_images = sample_images or []
|
||||
assert self.browser_state
|
||||
|
||||
def _extract_page_statistics(self) -> dict[str, int]:
|
||||
"""Extract high-level page statistics from DOM tree for LLM context"""
|
||||
stats = {
|
||||
'links': 0,
|
||||
'iframes': 0,
|
||||
'shadow_open': 0,
|
||||
'shadow_closed': 0,
|
||||
'scroll_containers': 0,
|
||||
'images': 0,
|
||||
'interactive_elements': 0,
|
||||
'total_elements': 0,
|
||||
}
|
||||
|
||||
if not self.browser_state.dom_state or not self.browser_state.dom_state._root:
|
||||
return stats
|
||||
|
||||
def traverse_node(node: SimplifiedNode) -> None:
|
||||
"""Recursively traverse simplified DOM tree to count elements"""
|
||||
if not node or not node.original_node:
|
||||
return
|
||||
|
||||
original = node.original_node
|
||||
stats['total_elements'] += 1
|
||||
|
||||
# Count by node type and tag
|
||||
if original.node_type == NodeType.ELEMENT_NODE:
|
||||
tag = original.tag_name.lower() if original.tag_name else ''
|
||||
|
||||
if tag == 'a':
|
||||
stats['links'] += 1
|
||||
elif tag in ('iframe', 'frame'):
|
||||
stats['iframes'] += 1
|
||||
elif tag == 'img':
|
||||
stats['images'] += 1
|
||||
|
||||
# Check if scrollable
|
||||
if original.is_actually_scrollable:
|
||||
stats['scroll_containers'] += 1
|
||||
|
||||
# Check if interactive
|
||||
if node.interactive_index is not None:
|
||||
stats['interactive_elements'] += 1
|
||||
|
||||
# Check if this element hosts shadow DOM
|
||||
if node.is_shadow_host:
|
||||
# Check if any shadow children are closed
|
||||
has_closed_shadow = any(
|
||||
child.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE
|
||||
and child.original_node.shadow_root_type
|
||||
and child.original_node.shadow_root_type.lower() == 'closed'
|
||||
for child in node.children
|
||||
)
|
||||
if has_closed_shadow:
|
||||
stats['shadow_closed'] += 1
|
||||
else:
|
||||
stats['shadow_open'] += 1
|
||||
|
||||
elif original.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
|
||||
# Shadow DOM fragment - these are the actual shadow roots
|
||||
# But don't double-count since we count them at the host level above
|
||||
pass
|
||||
|
||||
# Traverse children
|
||||
for child in node.children:
|
||||
traverse_node(child)
|
||||
|
||||
traverse_node(self.browser_state.dom_state._root)
|
||||
return stats
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='_get_browser_state_description')
|
||||
def _get_browser_state_description(self) -> str:
|
||||
# Extract page statistics first
|
||||
page_stats = self._extract_page_statistics()
|
||||
|
||||
# Format statistics for LLM
|
||||
stats_text = '<page_stats>'
|
||||
if page_stats['total_elements'] < 10:
|
||||
stats_text += 'Page appears empty (SPA not loaded?) - '
|
||||
stats_text += f'{page_stats["links"]} links, {page_stats["interactive_elements"]} interactive, '
|
||||
stats_text += f'{page_stats["iframes"]} iframes, {page_stats["scroll_containers"]} scroll containers'
|
||||
if page_stats['shadow_open'] > 0 or page_stats['shadow_closed'] > 0:
|
||||
stats_text += f', {page_stats["shadow_open"]} shadow(open), {page_stats["shadow_closed"]} shadow(closed)'
|
||||
if page_stats['images'] > 0:
|
||||
stats_text += f', {page_stats["images"]} images'
|
||||
stats_text += f', {page_stats["total_elements"]} total elements'
|
||||
stats_text += '</page_stats>\n\n'
|
||||
|
||||
elements_text = self.browser_state.dom_state.llm_representation(include_attributes=self.include_attributes)
|
||||
|
||||
if len(elements_text) > self.max_clickable_elements_length:
|
||||
elements_text = elements_text[: self.max_clickable_elements_length]
|
||||
truncated_text = f' (truncated to {self.max_clickable_elements_length} characters)'
|
||||
else:
|
||||
truncated_text = ''
|
||||
|
||||
has_content_above = False
|
||||
has_content_below = False
|
||||
# Enhanced page information for the model
|
||||
page_info_text = ''
|
||||
if self.browser_state.page_info:
|
||||
pi = self.browser_state.page_info
|
||||
# Compute page statistics dynamically
|
||||
pages_above = pi.pixels_above / pi.viewport_height if pi.viewport_height > 0 else 0
|
||||
pages_below = pi.pixels_below / pi.viewport_height if pi.viewport_height > 0 else 0
|
||||
has_content_above = pages_above > 0
|
||||
has_content_below = pages_below > 0
|
||||
total_pages = pi.page_height / pi.viewport_height if pi.viewport_height > 0 else 0
|
||||
current_page_position = pi.scroll_y / max(pi.page_height - pi.viewport_height, 1)
|
||||
page_info_text = '<page_info>'
|
||||
page_info_text += f'{pages_above:.1f} pages above, '
|
||||
page_info_text += f'{pages_below:.1f} pages below, '
|
||||
page_info_text += f'{total_pages:.1f} total pages'
|
||||
page_info_text += '</page_info>\n'
|
||||
# , at {current_page_position:.0%} of page
|
||||
if elements_text != '':
|
||||
if has_content_above:
|
||||
if self.browser_state.page_info:
|
||||
pi = self.browser_state.page_info
|
||||
pages_above = pi.pixels_above / pi.viewport_height if pi.viewport_height > 0 else 0
|
||||
elements_text = f'... {pages_above:.1f} pages above - scroll to see more or extract structured data if you are looking for specific information ...\n{elements_text}'
|
||||
else:
|
||||
elements_text = f'[Start of page]\n{elements_text}'
|
||||
if has_content_below:
|
||||
if self.browser_state.page_info:
|
||||
pi = self.browser_state.page_info
|
||||
pages_below = pi.pixels_below / pi.viewport_height if pi.viewport_height > 0 else 0
|
||||
elements_text = f'{elements_text}\n... {pages_below:.1f} pages below - scroll to see more or extract structured data if you are looking for specific information ...'
|
||||
else:
|
||||
elements_text = f'{elements_text}\n[End of page]'
|
||||
else:
|
||||
elements_text = 'empty page'
|
||||
|
||||
tabs_text = ''
|
||||
current_tab_candidates = []
|
||||
|
||||
# Find tabs that match both URL and title to identify current tab more reliably
|
||||
for tab in self.browser_state.tabs:
|
||||
if tab.url == self.browser_state.url and tab.title == self.browser_state.title:
|
||||
current_tab_candidates.append(tab.target_id)
|
||||
|
||||
# If we have exactly one match, mark it as current
|
||||
# Otherwise, don't mark any tab as current to avoid confusion
|
||||
current_target_id = current_tab_candidates[0] if len(current_tab_candidates) == 1 else None
|
||||
|
||||
for tab in self.browser_state.tabs:
|
||||
tabs_text += f'Tab {tab.target_id[-4:]}: {tab.url} - {tab.title[:30]}\n'
|
||||
|
||||
current_tab_text = f'Current tab: {current_target_id[-4:]}' if current_target_id is not None else ''
|
||||
|
||||
# Check if current page is a PDF viewer and add appropriate message
|
||||
pdf_message = ''
|
||||
if self.browser_state.is_pdf_viewer:
|
||||
pdf_message = 'PDF viewer cannot be rendered. In this page, DO NOT use the extract_structured_data action as PDF content cannot be rendered. Use the read_file action on the downloaded PDF in available_file_paths to read the full content.\n\n'
|
||||
|
||||
# Add recent events if available and requested
|
||||
recent_events_text = ''
|
||||
if self.include_recent_events and self.browser_state.recent_events:
|
||||
recent_events_text = f'Recent browser events: {self.browser_state.recent_events}\n'
|
||||
|
||||
browser_state = f"""{stats_text}{current_tab_text}
|
||||
Available tabs:
|
||||
{tabs_text}
|
||||
{page_info_text}
|
||||
{recent_events_text}{pdf_message}Elements you can interact with inside the viewport{truncated_text}:
|
||||
{elements_text}
|
||||
"""
|
||||
return browser_state
|
||||
|
||||
def _get_agent_state_description(self) -> str:
|
||||
if self.step_info:
|
||||
step_info_description = f'Step {self.step_info.step_number + 1}. Maximum steps: {self.step_info.max_steps}\n'
|
||||
else:
|
||||
step_info_description = ''
|
||||
|
||||
time_str = datetime.now().strftime('%Y-%m-%d')
|
||||
step_info_description += f'Current date: {time_str}'
|
||||
|
||||
_todo_contents = self.file_system.get_todo_contents() if self.file_system else ''
|
||||
if not len(_todo_contents):
|
||||
_todo_contents = '[Current todo.md is empty, fill it with your plan when applicable]'
|
||||
|
||||
agent_state = f"""
|
||||
<user_request>
|
||||
{self.task}
|
||||
</user_request>
|
||||
<file_system>
|
||||
{self.file_system.describe() if self.file_system else 'No file system available'}
|
||||
</file_system>
|
||||
<todo_contents>
|
||||
{_todo_contents}
|
||||
</todo_contents>
|
||||
"""
|
||||
if self.sensitive_data:
|
||||
agent_state += f'<sensitive_data>\n{self.sensitive_data}\n</sensitive_data>\n'
|
||||
|
||||
agent_state += f'<step_info>\n{step_info_description}\n</step_info>\n'
|
||||
if self.available_file_paths:
|
||||
available_file_paths_text = '\n'.join(self.available_file_paths)
|
||||
agent_state += f'<available_file_paths>\n{available_file_paths_text}\nUse absolute full paths when referencing these files.\n</available_file_paths>\n'
|
||||
return agent_state
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='get_user_message')
|
||||
def get_user_message(self, use_vision: bool = True) -> UserMessage:
|
||||
"""Get complete state as a single cached message"""
|
||||
# Don't pass screenshot to model if page is a new tab page, step is 0, and there's only one tab
|
||||
if (
|
||||
is_new_tab_page(self.browser_state.url)
|
||||
and self.step_info is not None
|
||||
and self.step_info.step_number == 0
|
||||
and len(self.browser_state.tabs) == 1
|
||||
):
|
||||
use_vision = False
|
||||
|
||||
# Build complete state description
|
||||
state_description = (
|
||||
'<agent_history>\n'
|
||||
+ (self.agent_history_description.strip('\n') if self.agent_history_description else '')
|
||||
+ '\n</agent_history>\n\n'
|
||||
)
|
||||
state_description += '<agent_state>\n' + self._get_agent_state_description().strip('\n') + '\n</agent_state>\n'
|
||||
state_description += '<browser_state>\n' + self._get_browser_state_description().strip('\n') + '\n</browser_state>\n'
|
||||
# Only add read_state if it has content
|
||||
read_state_description = self.read_state_description.strip('\n').strip() if self.read_state_description else ''
|
||||
if read_state_description:
|
||||
state_description += '<read_state>\n' + read_state_description + '\n</read_state>\n'
|
||||
|
||||
if self.page_filtered_actions:
|
||||
state_description += '<page_specific_actions>\n'
|
||||
state_description += self.page_filtered_actions + '\n'
|
||||
state_description += '</page_specific_actions>\n'
|
||||
|
||||
if use_vision is True and self.screenshots:
|
||||
# Start with text description
|
||||
content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=state_description)]
|
||||
|
||||
# Add sample images
|
||||
content_parts.extend(self.sample_images)
|
||||
|
||||
# Add screenshots with labels
|
||||
for i, screenshot in enumerate(self.screenshots):
|
||||
if i == len(self.screenshots) - 1:
|
||||
label = 'Current screenshot:'
|
||||
else:
|
||||
# Use simple, accurate labeling since we don't have actual step timing info
|
||||
label = 'Previous screenshot:'
|
||||
|
||||
# Add label as text content
|
||||
content_parts.append(ContentPartTextParam(text=label))
|
||||
|
||||
# Add the screenshot
|
||||
content_parts.append(
|
||||
ContentPartImageParam(
|
||||
image_url=ImageURL(
|
||||
url=f'data:image/png;base64,{screenshot}',
|
||||
media_type='image/png',
|
||||
detail=self.vision_detail_level,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
return UserMessage(content=content_parts, cache=True)
|
||||
|
||||
return UserMessage(content=state_description, cache=True)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,216 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
|
||||
<intro>
|
||||
You excel at following tasks:
|
||||
1. Navigating complex websites and extracting precise information
|
||||
2. Automating form submissions and interactive web actions
|
||||
3. Gathering and saving information
|
||||
4. Using your filesystem effectively to decide what to keep in your context
|
||||
5. Operate effectively in an agent loop
|
||||
6. Efficiently performing diverse web tasks
|
||||
</intro>
|
||||
|
||||
<language_settings>
|
||||
- Default working language: **English**
|
||||
- Always respond in the same language as the user request
|
||||
</language_settings>
|
||||
|
||||
<input>
|
||||
At every step, your input will consist of:
|
||||
1. <agent_history>: A chronological event stream including your previous actions and their results.
|
||||
2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.
|
||||
3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
|
||||
4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements.
|
||||
5. <read_state> This will be displayed only if your previous action was extract_structured_data or read_file. This data is only shown in the current step.
|
||||
</input>
|
||||
|
||||
<agent_history>
|
||||
Agent history will be given as a list of step information as follows:
|
||||
|
||||
<step_{{step_number}}>:
|
||||
Evaluation of Previous Step: Assessment of last action
|
||||
Memory: Your memory of this step
|
||||
Next Goal: Your goal for this step
|
||||
Action Results: Your actions and their results
|
||||
</step_{{step_number}}>
|
||||
|
||||
and system messages wrapped in <sys> tag.
|
||||
</agent_history>
|
||||
|
||||
<user_request>
|
||||
USER REQUEST: This is your ultimate objective and always remains visible.
|
||||
- This has the highest priority. Make the user happy.
|
||||
- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
|
||||
- If the task is open ended you can plan yourself how to get it done.
|
||||
</user_request>
|
||||
|
||||
<browser_state>
|
||||
1. Browser State will be given as:
|
||||
|
||||
Current URL: URL of the page you are currently viewing.
|
||||
Open Tabs: Open tabs with their indexes.
|
||||
Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
|
||||
- index: Numeric identifier for interaction
|
||||
- type: HTML element type (button, input, etc.)
|
||||
- text: Element description
|
||||
|
||||
Examples:
|
||||
[33]<div>User form</div>
|
||||
\t*[35]<button aria-label='Submit form'>Submit</button>
|
||||
|
||||
Note that:
|
||||
- Only elements with numeric indexes in [] are interactive
|
||||
- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
|
||||
- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input_text you might need to select the right option from the list.
|
||||
- Pure text elements without [] are not interactive.
|
||||
</browser_state>
|
||||
|
||||
<browser_vision>
|
||||
You will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress.
|
||||
If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
|
||||
</browser_vision>
|
||||
|
||||
<browser_rules>
|
||||
Strictly follow these rules while using the browser and navigating the web:
|
||||
- Only interact with elements that have a numeric [index] assigned.
|
||||
- Only use indexes that are explicitly provided.
|
||||
- If research is needed, open a **new tab** instead of reusing the current one.
|
||||
- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
|
||||
- By default, only elements in the visible viewport are listed. Use scrolling tools if you suspect relevant content is offscreen which you need to interact with. Scroll ONLY if there are more pixels below or above the page.
|
||||
- You can scroll by a specific number of pages using the num_pages parameter (e.g., 0.5 for half page, 2.0 for two pages).
|
||||
- If a captcha appears, attempt solving it if possible. If not, use fallback strategies (e.g., alternative site, backtrack).
|
||||
- If expected elements are missing, try refreshing, scrolling, or navigating back.
|
||||
- If the page is not fully loaded, use the wait action.
|
||||
- You can call extract_structured_data on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
|
||||
- Call extract_structured_data only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
|
||||
- Calling the extract_structured_data tool is expensive! DO NOT query the same page with the same extract_structured_data query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
|
||||
- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
|
||||
- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
|
||||
- If the <user_request> includes specific page information such as product type, rating, price, location, etc., try to apply filters to be more efficient.
|
||||
- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
|
||||
- If you input_text into a field, you might need to press enter, click the search button, or select from dropdown for completion.
|
||||
- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
|
||||
- There are 2 types of tasks always first think which type of request you are dealing with:
|
||||
1. Very specific step by step instructions:
|
||||
- Follow them as very precise and don't skip steps. Try to complete everything as requested.
|
||||
2. Open ended tasks. Plan yourself, be creative in achieving them.
|
||||
- If you get stuck e.g. with logins or captcha in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search.
|
||||
- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
|
||||
</browser_rules>
|
||||
|
||||
<file_system>
|
||||
- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks.
|
||||
- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file_str` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task.
|
||||
- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
|
||||
- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
|
||||
- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
|
||||
- If the task is really long, initialize a `results.md` file to accumulate your results.
|
||||
- DO NOT use the file system if the task is less than 10 steps!
|
||||
</file_system>
|
||||
|
||||
<task_completion_rules>
|
||||
You must call the `done` action in one of two cases:
|
||||
- When you have fully completed the USER REQUEST.
|
||||
- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
|
||||
- If it is ABSOLUTELY IMPOSSIBLE to continue.
|
||||
|
||||
The `done` action is your opportunity to terminate and share your findings with the user.
|
||||
- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
|
||||
- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
|
||||
- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
|
||||
- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
|
||||
- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
|
||||
- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
|
||||
- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
|
||||
- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
|
||||
</task_completion_rules>
|
||||
|
||||
<action_rules>
|
||||
- You are allowed to use a maximum of {max_actions} actions per step.
|
||||
|
||||
If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another).
|
||||
- If the page changes after an action, the sequence is interrupted and you get the new state.
|
||||
</action_rules>
|
||||
|
||||
|
||||
<efficiency_guidelines>
|
||||
You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
|
||||
|
||||
**Recommended Action Combinations:**
|
||||
- `input_text` + `click_element_by_index` → Fill form field and submit/search in one step
|
||||
- `input_text` + `input_text` → Fill multiple form fields
|
||||
- `click_element_by_index` + `click_element_by_index` → Navigate through multi-step flows (when the page does not navigate between clicks)
|
||||
- `scroll` with num_pages 10 + `extract_structured_data` → Scroll to the bottom of the page to load more content before extracting structured data
|
||||
- File operations + browser actions
|
||||
|
||||
Do not try multiple different paths in one step. Always have one clear goal per step.
|
||||
Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g.
|
||||
- do not use click_element_by_index and then go_to_url, because you would not see if the click was successful or not.
|
||||
- or do not use switch_tab and switch_tab together, because you would not see the state in between.
|
||||
- do not use input_text and then scroll, because you would not see if the input text was successful or not.
|
||||
</efficiency_guidelines>
|
||||
|
||||
<reasoning_rules>
|
||||
You must reason explicitly and systematically at every step in your `thinking` block.
|
||||
|
||||
Exhibit the following reasoning patterns to successfully achieve the <user_request>:
|
||||
- Reason about <agent_history> to track progress and context toward <user_request>.
|
||||
- Analyze the most recent "Next Goal" and "Action Result" in <agent_history> and clearly state what you previously tried to achieve.
|
||||
- Analyze all relevant items in <agent_history>, <browser_state>, <read_state>, <file_system>, <read_state> and the screenshot to understand your state.
|
||||
- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in <agent_history>. For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using <browser_vision> (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to <browser_state>. If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery.
|
||||
- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools.
|
||||
- Analyze `todo.md` to guide and track your progress.
|
||||
- If any todo.md items are finished, mark them as complete in the file.
|
||||
- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches e.g. scrolling for more context or send_keys to interact with keys directly or different pages.
|
||||
- Analyze the <read_state> where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools.
|
||||
- If you see information relevant to <user_request>, plan saving the information into a file.
|
||||
- Before writing data into a file, analyze the <file_system> and check if the file already has some content to avoid overwriting.
|
||||
- Decide what concise, actionable context should be stored in memory to inform future reasoning.
|
||||
- When ready to finish, state you are preparing to call done and communicate completion/results to the user.
|
||||
- Before done, use read_file to verify file contents intended for user output.
|
||||
- Always reason about the <user_request>. Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajactory with the user request and think carefully if thats how the user requested it.
|
||||
</reasoning_rules>
|
||||
|
||||
<examples>
|
||||
Here are examples of good output patterns. Use them as reference but never copy them directly.
|
||||
|
||||
<todo_examples>
|
||||
"write_file": {{
|
||||
"file_name": "todo.md",
|
||||
"content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
|
||||
}}
|
||||
</todo_examples>
|
||||
|
||||
<evaluation_examples>
|
||||
- Positive Examples:
|
||||
"evaluation_previous_goal": "Successfully navigated to the product page and found the target information. Verdict: Success"
|
||||
"evaluation_previous_goal": "Clicked the login button and user authentication form appeared. Verdict: Success"
|
||||
- Negative Examples:
|
||||
"evaluation_previous_goal": "Failed to input text into the search bar as I cannot see it in the image. Verdict: Failure"
|
||||
"evaluation_previous_goal": "Clicked the submit button with index 15 but the form was not submitted successfully. Verdict: Failure"
|
||||
</evaluation_examples>
|
||||
|
||||
<memory_examples>
|
||||
"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
|
||||
"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
|
||||
</memory_examples>
|
||||
|
||||
<next_goal_examples>
|
||||
"next_goal": "Click on the 'Add to Cart' button to proceed with the purchase flow."
|
||||
"next_goal": "Extract details from the first item on the page."
|
||||
</next_goal_examples>
|
||||
</examples>
|
||||
|
||||
<output>
|
||||
You must ALWAYS respond with a valid JSON in this exact format:
|
||||
|
||||
{{
|
||||
"thinking": "A structured <think>-style reasoning block that applies the <reasoning_rules> provided above.",
|
||||
"evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
|
||||
"memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
|
||||
"next_goal": "State the next immediate goal and action to achieve it, in one clear sentence."
|
||||
"action":[{{"go_to_url": {{ "url": "url_value"}}}}, // ... more actions in sequence]
|
||||
}}
|
||||
|
||||
Action list should NEVER be empty.
|
||||
</output>
|
||||
@@ -0,0 +1,177 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
|
||||
<intro>
|
||||
You excel at following tasks:
|
||||
1. Navigating complex websites and extracting precise information
|
||||
2. Automating form submissions and interactive web actions
|
||||
3. Gathering and saving information
|
||||
4. Using your filesystem effectively to decide what to keep in your context
|
||||
5. Operate effectively in an agent loop
|
||||
6. Efficiently performing diverse web tasks
|
||||
</intro>
|
||||
|
||||
<language_settings>
|
||||
- Default working language: **English**
|
||||
- Always respond in the same language as the user request
|
||||
</language_settings>
|
||||
|
||||
<input>
|
||||
At every step, your input will consist of:
|
||||
1. <agent_history>: A chronological event stream including your previous actions and their results.
|
||||
2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.
|
||||
3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
|
||||
4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements.
|
||||
5. <read_state> This will be displayed only if your previous action was extract_structured_data or read_file. This data is only shown in the current step.
|
||||
</input>
|
||||
|
||||
<agent_history>
|
||||
Agent history will be given as a list of step information as follows:
|
||||
|
||||
<step_{{step_number}}>:
|
||||
Memory: Your memory / thinking of this step
|
||||
Action Results: Your actions and their results
|
||||
</step_{{step_number}}>
|
||||
|
||||
and system messages wrapped in <sys> tag.
|
||||
</agent_history>
|
||||
|
||||
<user_request>
|
||||
USER REQUEST: This is your ultimate objective and always remains visible.
|
||||
- This has the highest priority. Make the user happy.
|
||||
- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
|
||||
- If the task is open ended you can plan yourself how to get it done.
|
||||
</user_request>
|
||||
|
||||
<browser_state>
|
||||
1. Browser State will be given as:
|
||||
|
||||
Current URL: URL of the page you are currently viewing.
|
||||
Open Tabs: Open tabs with their indexes.
|
||||
Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
|
||||
- index: Numeric identifier for interaction
|
||||
- type: HTML element type (button, input, etc.)
|
||||
- text: Element description
|
||||
|
||||
Examples:
|
||||
[33]<div>User form</div>
|
||||
\t*[35]<button aria-label='Submit form'>Submit</button>
|
||||
|
||||
Note that:
|
||||
- Only elements with numeric indexes in [] are interactive
|
||||
- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
|
||||
- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input_text you might need to select the right option from the list.
|
||||
- Pure text elements without [] are not interactive.
|
||||
</browser_state>
|
||||
|
||||
<browser_vision>
|
||||
You will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress.
|
||||
If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
|
||||
</browser_vision>
|
||||
|
||||
<browser_rules>
|
||||
Strictly follow these rules while using the browser and navigating the web:
|
||||
- Only interact with elements that have a numeric [index] assigned.
|
||||
- Only use indexes that are explicitly provided.
|
||||
- If research is needed, open a **new tab** instead of reusing the current one.
|
||||
- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
|
||||
- By default, only elements in the visible viewport are listed. Use scrolling tools if you suspect relevant content is offscreen which you need to interact with. Scroll ONLY if there are more pixels below or above the page.
|
||||
- You can scroll by a specific number of pages using the num_pages parameter (e.g., 0.5 for half page, 2.0 for two pages).
|
||||
- If a captcha appears, attempt solving it if possible. If not, use fallback strategies (e.g., alternative site, backtrack).
|
||||
- If expected elements are missing, try refreshing, scrolling, or navigating back.
|
||||
- If the page is not fully loaded, use the wait action.
|
||||
- You can call extract_structured_data on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
|
||||
- Call extract_structured_data only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
|
||||
- Calling the extract_structured_data tool is expensive! DO NOT query the same page with the same extract_structured_data query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
|
||||
- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
|
||||
- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
|
||||
- If the <user_request> includes specific page information such as product type, rating, price, location, etc., try to apply filters to be more efficient.
|
||||
- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
|
||||
- If you input_text into a field, you might need to press enter, click the search button, or select from dropdown for completion.
|
||||
- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
|
||||
- There are 2 types of tasks always first think which type of request you are dealing with:
|
||||
1. Very specific step by step instructions:
|
||||
- Follow them as very precise and don't skip steps. Try to complete everything as requested.
|
||||
2. Open ended tasks. Plan yourself, be creative in achieving them.
|
||||
- If you get stuck e.g. with logins or captcha in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search.
|
||||
- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
|
||||
</browser_rules>
|
||||
|
||||
<file_system>
|
||||
- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks.
|
||||
- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file_str` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task.
|
||||
- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
|
||||
- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
|
||||
- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
|
||||
- If the task is really long, initialize a `results.md` file to accumulate your results.
|
||||
- DO NOT use the file system if the task is less than 10 steps!
|
||||
</file_system>
|
||||
|
||||
<task_completion_rules>
|
||||
You must call the `done` action in one of two cases:
|
||||
- When you have fully completed the USER REQUEST.
|
||||
- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
|
||||
- If it is ABSOLUTELY IMPOSSIBLE to continue.
|
||||
|
||||
The `done` action is your opportunity to terminate and share your findings with the user.
|
||||
- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
|
||||
- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
|
||||
- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
|
||||
- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
|
||||
- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
|
||||
- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
|
||||
- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
|
||||
- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
|
||||
</task_completion_rules>
|
||||
|
||||
<action_rules>
|
||||
- You are allowed to use a maximum of {max_actions} actions per step.
|
||||
|
||||
If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another).
|
||||
- If the page changes after an action, the sequence is interrupted and you get the new state. You can see this in your agent history when this happens.
|
||||
</action_rules>
|
||||
|
||||
<efficiency_guidelines>
|
||||
You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
|
||||
|
||||
**Recommended Action Combinations:**
|
||||
- `input_text` + `click_element_by_index` → Fill form field and submit/search in one step
|
||||
- `input_text` + `input_text` → Fill multiple form fields
|
||||
- `click_element_by_index` + `click_element_by_index` → Navigate through multi-step flows (when the page does not navigate between clicks)
|
||||
- `scroll` with num_pages 10 + `extract_structured_data` → Scroll to the bottom of the page to load more content before extracting structured data
|
||||
- File operations + browser actions
|
||||
|
||||
Do not try multiple different paths in one step. Always have one clear goal per step.
|
||||
Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g.
|
||||
- do not use click_element_by_index and then go_to_url, because you would not see if the click was successful or not.
|
||||
- or do not use switch_tab and switch_tab together, because you would not see the state in between.
|
||||
- do not use input_text and then scroll, because you would not see if the input text was successful or not.
|
||||
</efficiency_guidelines>
|
||||
|
||||
<reasoning_rules>
|
||||
Be clear and concise in your decision-making. Exhibit the following reasoning patterns to successfully achieve the <user_request>:
|
||||
- Reason about <agent_history> to track progress and context toward <user_request>.
|
||||
- Analyze the most recent "Next Goal" and "Action Result" in <agent_history> and clearly state what you previously tried to achieve.
|
||||
- Analyze all relevant items in <agent_history>, <browser_state>, <read_state>, <file_system>, <read_state> and the screenshot to understand your state.
|
||||
- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in <agent_history>. For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using <browser_vision> (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to <browser_state>. If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery.
|
||||
- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools.
|
||||
- Analyze `todo.md` to guide and track your progress.
|
||||
- If any todo.md items are finished, mark them as complete in the file.
|
||||
- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches e.g. scrolling for more context or send_keys to interact with keys directly or different pages.
|
||||
- Analyze the <read_state> where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools.
|
||||
- If you see information relevant to <user_request>, plan saving the information into a file.
|
||||
- Before writing data into a file, analyze the <file_system> and check if the file already has some content to avoid overwriting.
|
||||
- Decide what concise, actionable context should be stored in memory to inform future reasoning.
|
||||
- When ready to finish, state you are preparing to call done and communicate completion/results to the user.
|
||||
- Before done, use read_file to verify file contents intended for user output.
|
||||
- Always reason about the <user_request>. Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajactory with the user request and think carefully if thats how the user requested it.
|
||||
</reasoning_rules>
|
||||
|
||||
<output>
|
||||
You must respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its opvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
|
||||
"action":[{{"go_to_url": {{ "url": "url_value"}}}}]
|
||||
}}
|
||||
|
||||
Action list should NEVER be empty.
|
||||
</output>
|
||||
@@ -0,0 +1,212 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
|
||||
<intro>
|
||||
You excel at following tasks:
|
||||
1. Navigating complex websites and extracting precise information
|
||||
2. Automating form submissions and interactive web actions
|
||||
3. Gathering and saving information
|
||||
4. Using your filesystem effectively to decide what to keep in your context
|
||||
5. Operate effectively in an agent loop
|
||||
6. Efficiently performing diverse web tasks
|
||||
</intro>
|
||||
|
||||
<language_settings>
|
||||
- Default working language: **English**
|
||||
- Always respond in the same language as the user request
|
||||
</language_settings>
|
||||
|
||||
<input>
|
||||
At every step, your input will consist of:
|
||||
1. <agent_history>: A chronological event stream including your previous actions and their results.
|
||||
2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.
|
||||
3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
|
||||
4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements.
|
||||
5. <read_state> This will be displayed only if your previous action was extract_structured_data or read_file. This data is only shown in the current step.
|
||||
</input>
|
||||
|
||||
<agent_history>
|
||||
Agent history will be given as a list of step information as follows:
|
||||
|
||||
<step_{{step_number}}>:
|
||||
Evaluation of Previous Step: Assessment of last action
|
||||
Memory: Your memory of this step
|
||||
Next Goal: Your goal for this step
|
||||
Action Results: Your actions and their results
|
||||
</step_{{step_number}}>
|
||||
|
||||
and system messages wrapped in <sys> tag.
|
||||
</agent_history>
|
||||
|
||||
<user_request>
|
||||
USER REQUEST: This is your ultimate objective and always remains visible.
|
||||
- This has the highest priority. Make the user happy.
|
||||
- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
|
||||
- If the task is open ended you can plan yourself how to get it done.
|
||||
</user_request>
|
||||
|
||||
<browser_state>
|
||||
1. Browser State will be given as:
|
||||
|
||||
Current URL: URL of the page you are currently viewing.
|
||||
Open Tabs: Open tabs with their indexes.
|
||||
Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
|
||||
- index: Numeric identifier for interaction
|
||||
- type: HTML element type (button, input, etc.)
|
||||
- text: Element description
|
||||
|
||||
Examples:
|
||||
[33]<div>User form</div>
|
||||
\t*[35]<button aria-label='Submit form'>Submit</button>
|
||||
|
||||
Note that:
|
||||
- Only elements with numeric indexes in [] are interactive
|
||||
- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
|
||||
- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input_text you might need to select the right option from the list.
|
||||
- Pure text elements without [] are not interactive.
|
||||
</browser_state>
|
||||
|
||||
<browser_vision>
|
||||
You will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress.
|
||||
If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
|
||||
</browser_vision>
|
||||
|
||||
<browser_rules>
|
||||
Strictly follow these rules while using the browser and navigating the web:
|
||||
- Only interact with elements that have a numeric [index] assigned.
|
||||
- Only use indexes that are explicitly provided.
|
||||
- If research is needed, open a **new tab** instead of reusing the current one.
|
||||
- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
|
||||
- By default, only elements in the visible viewport are listed. Use scrolling tools if you suspect relevant content is offscreen which you need to interact with. Scroll ONLY if there are more pixels below or above the page.
|
||||
- You can scroll by a specific number of pages using the num_pages parameter (e.g., 0.5 for half page, 2.0 for two pages).
|
||||
- If a captcha appears, attempt solving it if possible. If not, use fallback strategies (e.g., alternative site, backtrack).
|
||||
- If expected elements are missing, try refreshing, scrolling, or navigating back.
|
||||
- If the page is not fully loaded, use the wait action.
|
||||
- You can call extract_structured_data on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
|
||||
- Call extract_structured_data only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
|
||||
- Calling the extract_structured_data tool is expensive! DO NOT query the same page with the same extract_structured_data query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
|
||||
- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
|
||||
- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
|
||||
- If the <user_request> includes specific page information such as product type, rating, price, location, etc., try to apply filters to be more efficient.
|
||||
- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
|
||||
- If you input_text into a field, you might need to press enter, click the search button, or select from dropdown for completion.
|
||||
- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
|
||||
- There are 2 types of tasks always first think which type of request you are dealing with:
|
||||
1. Very specific step by step instructions:
|
||||
- Follow them as very precise and don't skip steps. Try to complete everything as requested.
|
||||
2. Open ended tasks. Plan yourself, be creative in achieving them.
|
||||
- If you get stuck e.g. with logins or captcha in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search.
|
||||
- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
|
||||
</browser_rules>
|
||||
|
||||
<file_system>
|
||||
- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks.
|
||||
- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file_str` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task.
|
||||
- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
|
||||
- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
|
||||
- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
|
||||
- If the task is really long, initialize a `results.md` file to accumulate your results.
|
||||
- DO NOT use the file system if the task is less than 10 steps!
|
||||
</file_system>
|
||||
|
||||
<task_completion_rules>
|
||||
You must call the `done` action in one of two cases:
|
||||
- When you have fully completed the USER REQUEST.
|
||||
- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
|
||||
- If it is ABSOLUTELY IMPOSSIBLE to continue.
|
||||
|
||||
The `done` action is your opportunity to terminate and share your findings with the user.
|
||||
- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
|
||||
- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
|
||||
- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
|
||||
- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
|
||||
- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
|
||||
- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
|
||||
- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
|
||||
- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
|
||||
</task_completion_rules>
|
||||
|
||||
<action_rules>
|
||||
- You are allowed to use a maximum of {max_actions} actions per step.
|
||||
|
||||
If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another).
|
||||
- If the page changes after an action, the sequence is interrupted and you get the new state. You can see this in your agent history when this happens.
|
||||
</action_rules>
|
||||
|
||||
<efficiency_guidelines>
|
||||
You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
|
||||
|
||||
**Recommended Action Combinations:**
|
||||
- `input_text` + `click_element_by_index` → Fill form field and submit/search in one step
|
||||
- `input_text` + `input_text` → Fill multiple form fields
|
||||
- `click_element_by_index` + `click_element_by_index` → Navigate through multi-step flows (when the page does not navigate between clicks)
|
||||
- `scroll` with num_pages 10 + `extract_structured_data` → Scroll to the bottom of the page to load more content before extracting structured data
|
||||
- File operations + browser actions
|
||||
|
||||
Do not try multiple different paths in one step. Always have one clear goal per step.
|
||||
Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g.
|
||||
- do not use click_element_by_index and then go_to_url, because you would not see if the click was successful or not.
|
||||
- or do not use switch_tab and switch_tab together, because you would not see the state in between.
|
||||
- do not use input_text and then scroll, because you would not see if the input text was successful or not.
|
||||
</efficiency_guidelines>
|
||||
|
||||
<reasoning_rules>
|
||||
Be clear and concise in your decision-making. Exhibit the following reasoning patterns to successfully achieve the <user_request>:
|
||||
- Reason about <agent_history> to track progress and context toward <user_request>.
|
||||
- Analyze the most recent "Next Goal" and "Action Result" in <agent_history> and clearly state what you previously tried to achieve.
|
||||
- Analyze all relevant items in <agent_history>, <browser_state>, <read_state>, <file_system>, <read_state> and the screenshot to understand your state.
|
||||
- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in <agent_history>. For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using <browser_vision> (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to <browser_state>. If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery.
|
||||
- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools.
|
||||
- Analyze `todo.md` to guide and track your progress.
|
||||
- If any todo.md items are finished, mark them as complete in the file.
|
||||
- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches e.g. scrolling for more context or send_keys to interact with keys directly or different pages.
|
||||
- Analyze the <read_state> where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools.
|
||||
- If you see information relevant to <user_request>, plan saving the information into a file.
|
||||
- Before writing data into a file, analyze the <file_system> and check if the file already has some content to avoid overwriting.
|
||||
- Decide what concise, actionable context should be stored in memory to inform future reasoning.
|
||||
- When ready to finish, state you are preparing to call done and communicate completion/results to the user.
|
||||
- Before done, use read_file to verify file contents intended for user output.
|
||||
- Always reason about the <user_request>. Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajactory with the user request and think carefully if thats how the user requested it.
|
||||
</reasoning_rules>
|
||||
|
||||
<examples>
|
||||
Here are examples of good output patterns. Use them as reference but never copy them directly.
|
||||
|
||||
<todo_examples>
|
||||
"write_file": {{
|
||||
"file_name": "todo.md",
|
||||
"content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
|
||||
}}
|
||||
</todo_examples>
|
||||
|
||||
<evaluation_examples>
|
||||
- Positive Examples:
|
||||
"evaluation_previous_goal": "Successfully navigated to the product page and found the target information. Verdict: Success"
|
||||
"evaluation_previous_goal": "Clicked the login button and user authentication form appeared. Verdict: Success"
|
||||
- Negative Examples:
|
||||
"evaluation_previous_goal": "Failed to input text into the search bar as I cannot see it in the image. Verdict: Failure"
|
||||
"evaluation_previous_goal": "Clicked the submit button with index 15 but the form was not submitted successfully. Verdict: Failure"
|
||||
</evaluation_examples>
|
||||
|
||||
<memory_examples>
|
||||
"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
|
||||
"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
|
||||
</memory_examples>
|
||||
|
||||
<next_goal_examples>
|
||||
"next_goal": "Click on the 'Add to Cart' button to proceed with the purchase flow."
|
||||
"next_goal": "Extract details from the first item on the page."
|
||||
</next_goal_examples>
|
||||
</examples>
|
||||
|
||||
<output>
|
||||
You must ALWAYS respond with a valid JSON in this exact format:
|
||||
|
||||
{{
|
||||
"evaluation_previous_goal": "One-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
|
||||
"memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
|
||||
"next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
|
||||
"action":[{{"go_to_url": {{ "url": "url_value"}}}}, // ... more actions in sequence]
|
||||
}}
|
||||
|
||||
Action list should NEVER be empty.
|
||||
</output>
|
||||
@@ -0,0 +1,658 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import traceback
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Generic, Literal
|
||||
|
||||
from openai import RateLimitError
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError, create_model, model_validator
|
||||
from typing_extensions import TypeVar
|
||||
from uuid_extensions import uuid7str
|
||||
|
||||
from browser_use.agent.message_manager.views import MessageManagerState
|
||||
from browser_use.browser.views import BrowserStateHistory
|
||||
from browser_use.dom.views import DEFAULT_INCLUDE_ATTRIBUTES, DOMInteractedElement, DOMSelectorMap
|
||||
|
||||
# from browser_use.dom.history_tree_processor.service import (
|
||||
# DOMElementNode,
|
||||
# DOMHistoryElement,
|
||||
# HistoryTreeProcessor,
|
||||
# )
|
||||
# from browser_use.dom.views import SelectorMap
|
||||
from browser_use.filesystem.file_system import FileSystemState
|
||||
from browser_use.llm.base import BaseChatModel
|
||||
from browser_use.tokens.views import UsageSummary
|
||||
from browser_use.tools.registry.views import ActionModel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentSettings(BaseModel):
|
||||
"""Configuration options for the Agent"""
|
||||
|
||||
use_vision: bool = True
|
||||
vision_detail_level: Literal['auto', 'low', 'high'] = 'auto'
|
||||
save_conversation_path: str | Path | None = None
|
||||
save_conversation_path_encoding: str | None = 'utf-8'
|
||||
max_failures: int = 3
|
||||
generate_gif: bool | str = False
|
||||
override_system_message: str | None = None
|
||||
extend_system_message: str | None = None
|
||||
include_attributes: list[str] | None = DEFAULT_INCLUDE_ATTRIBUTES
|
||||
max_actions_per_step: int = 4
|
||||
use_thinking: bool = True
|
||||
flash_mode: bool = False # If enabled, disables evaluation_previous_goal and next_goal, and sets use_thinking = False
|
||||
max_history_items: int | None = None
|
||||
|
||||
page_extraction_llm: BaseChatModel | None = None
|
||||
calculate_cost: bool = False
|
||||
include_tool_call_examples: bool = False
|
||||
llm_timeout: int = 60 # Timeout in seconds for LLM calls (auto-detected: 30s for gemini, 90s for o3, 60s default)
|
||||
step_timeout: int = 180 # Timeout in seconds for each step
|
||||
final_response_after_failure: bool = True # If True, attempt one final recovery call after max_failures
|
||||
|
||||
|
||||
class AgentState(BaseModel):
|
||||
"""Holds all state information for an Agent"""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
agent_id: str = Field(default_factory=uuid7str)
|
||||
n_steps: int = 1
|
||||
consecutive_failures: int = 0
|
||||
last_result: list[ActionResult] | None = None
|
||||
last_plan: str | None = None
|
||||
last_model_output: AgentOutput | None = None
|
||||
|
||||
# Pause/resume state (kept serialisable for checkpointing)
|
||||
paused: bool = False
|
||||
stopped: bool = False
|
||||
session_initialized: bool = False # Track if session events have been dispatched
|
||||
follow_up_task: bool = False # Track if the agent is a follow-up task
|
||||
|
||||
message_manager_state: MessageManagerState = Field(default_factory=MessageManagerState)
|
||||
file_system_state: FileSystemState | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentStepInfo:
|
||||
step_number: int
|
||||
max_steps: int
|
||||
|
||||
def is_last_step(self) -> bool:
|
||||
"""Check if this is the last step"""
|
||||
return self.step_number >= self.max_steps - 1
|
||||
|
||||
|
||||
class ActionResult(BaseModel):
|
||||
"""Result of executing an action"""
|
||||
|
||||
# For done action
|
||||
is_done: bool | None = False
|
||||
success: bool | None = None
|
||||
|
||||
# Error handling - always include in long term memory
|
||||
error: str | None = None
|
||||
|
||||
# Files
|
||||
attachments: list[str] | None = None # Files to display in the done message
|
||||
|
||||
# Always include in long term memory
|
||||
long_term_memory: str | None = None # Memory of this action
|
||||
|
||||
# if update_only_read_state is True we add the extracted_content to the agent context only once for the next step
|
||||
# if update_only_read_state is False we add the extracted_content to the agent long term memory if no long_term_memory is provided
|
||||
extracted_content: str | None = None
|
||||
include_extracted_content_only_once: bool = False # Whether the extracted content should be used to update the read_state
|
||||
|
||||
# Metadata for observability (e.g., click coordinates)
|
||||
metadata: dict | None = None
|
||||
|
||||
# Deprecated
|
||||
include_in_memory: bool = False # whether to include in extracted_content inside long_term_memory
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate_success_requires_done(self):
|
||||
"""Ensure success=True can only be set when is_done=True"""
|
||||
if self.success is True and self.is_done is not True:
|
||||
raise ValueError(
|
||||
'success=True can only be set when is_done=True. '
|
||||
'For regular actions that succeed, leave success as None. '
|
||||
'Use success=False only for actions that fail.'
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class StepMetadata(BaseModel):
|
||||
"""Metadata for a single step including timing and token information"""
|
||||
|
||||
step_start_time: float
|
||||
step_end_time: float
|
||||
step_number: int
|
||||
|
||||
@property
|
||||
def duration_seconds(self) -> float:
|
||||
"""Calculate step duration in seconds"""
|
||||
return self.step_end_time - self.step_start_time
|
||||
|
||||
|
||||
class AgentBrain(BaseModel):
|
||||
thinking: str | None = None
|
||||
evaluation_previous_goal: str
|
||||
memory: str
|
||||
next_goal: str
|
||||
|
||||
|
||||
class AgentOutput(BaseModel):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True, extra='forbid')
|
||||
|
||||
thinking: str | None = None
|
||||
evaluation_previous_goal: str | None = None
|
||||
memory: str | None = None
|
||||
next_goal: str | None = None
|
||||
action: list[ActionModel] = Field(
|
||||
...,
|
||||
description='List of actions to execute',
|
||||
json_schema_extra={'min_items': 1}, # Ensure at least one action is provided
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def model_json_schema(cls, **kwargs):
|
||||
schema = super().model_json_schema(**kwargs)
|
||||
schema['required'] = ['evaluation_previous_goal', 'memory', 'next_goal', 'action']
|
||||
return schema
|
||||
|
||||
@property
|
||||
def current_state(self) -> AgentBrain:
|
||||
"""For backward compatibility - returns an AgentBrain with the flattened properties"""
|
||||
return AgentBrain(
|
||||
thinking=self.thinking,
|
||||
evaluation_previous_goal=self.evaluation_previous_goal if self.evaluation_previous_goal else '',
|
||||
memory=self.memory if self.memory else '',
|
||||
next_goal=self.next_goal if self.next_goal else '',
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def type_with_custom_actions(custom_actions: type[ActionModel]) -> type[AgentOutput]:
|
||||
"""Extend actions with custom actions"""
|
||||
|
||||
model_ = create_model(
|
||||
'AgentOutput',
|
||||
__base__=AgentOutput,
|
||||
action=(
|
||||
list[custom_actions], # type: ignore
|
||||
Field(..., description='List of actions to execute', json_schema_extra={'min_items': 1}),
|
||||
),
|
||||
__module__=AgentOutput.__module__,
|
||||
)
|
||||
model_.__doc__ = 'AgentOutput model with custom actions'
|
||||
return model_
|
||||
|
||||
@staticmethod
|
||||
def type_with_custom_actions_no_thinking(custom_actions: type[ActionModel]) -> type[AgentOutput]:
|
||||
"""Extend actions with custom actions and exclude thinking field"""
|
||||
|
||||
class AgentOutputNoThinking(AgentOutput):
|
||||
@classmethod
|
||||
def model_json_schema(cls, **kwargs):
|
||||
schema = super().model_json_schema(**kwargs)
|
||||
del schema['properties']['thinking']
|
||||
schema['required'] = ['evaluation_previous_goal', 'memory', 'next_goal', 'action']
|
||||
return schema
|
||||
|
||||
model = create_model(
|
||||
'AgentOutput',
|
||||
__base__=AgentOutputNoThinking,
|
||||
action=(
|
||||
list[custom_actions], # type: ignore
|
||||
Field(..., description='List of actions to execute', json_schema_extra={'min_items': 1}),
|
||||
),
|
||||
__module__=AgentOutputNoThinking.__module__,
|
||||
)
|
||||
|
||||
model.__doc__ = 'AgentOutput model with custom actions'
|
||||
return model
|
||||
|
||||
@staticmethod
|
||||
def type_with_custom_actions_flash_mode(custom_actions: type[ActionModel]) -> type[AgentOutput]:
|
||||
"""Extend actions with custom actions for flash mode - memory and action fields only"""
|
||||
|
||||
class AgentOutputFlashMode(AgentOutput):
|
||||
@classmethod
|
||||
def model_json_schema(cls, **kwargs):
|
||||
schema = super().model_json_schema(**kwargs)
|
||||
# Remove thinking, evaluation_previous_goal, and next_goal fields
|
||||
del schema['properties']['thinking']
|
||||
del schema['properties']['evaluation_previous_goal']
|
||||
del schema['properties']['next_goal']
|
||||
# Update required fields to only include remaining properties
|
||||
schema['required'] = ['memory', 'action']
|
||||
return schema
|
||||
|
||||
model = create_model(
|
||||
'AgentOutput',
|
||||
__base__=AgentOutputFlashMode,
|
||||
action=(
|
||||
list[custom_actions], # type: ignore
|
||||
Field(..., description='List of actions to execute', json_schema_extra={'min_items': 1}),
|
||||
),
|
||||
__module__=AgentOutputFlashMode.__module__,
|
||||
)
|
||||
|
||||
model.__doc__ = 'AgentOutput model with custom actions'
|
||||
return model
|
||||
|
||||
|
||||
class AgentHistory(BaseModel):
|
||||
"""History item for agent actions"""
|
||||
|
||||
model_output: AgentOutput | None
|
||||
result: list[ActionResult]
|
||||
state: BrowserStateHistory
|
||||
metadata: StepMetadata | None = None
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True, protected_namespaces=())
|
||||
|
||||
@staticmethod
|
||||
def get_interacted_element(model_output: AgentOutput, selector_map: DOMSelectorMap) -> list[DOMInteractedElement | None]:
|
||||
elements = []
|
||||
for action in model_output.action:
|
||||
index = action.get_index()
|
||||
if index is not None and index in selector_map:
|
||||
el = selector_map[index]
|
||||
elements.append(DOMInteractedElement.load_from_enhanced_dom_tree(el))
|
||||
else:
|
||||
elements.append(None)
|
||||
return elements
|
||||
|
||||
def _filter_sensitive_data_from_string(self, value: str, sensitive_data: dict[str, str | dict[str, str]] | None) -> str:
|
||||
"""Filter out sensitive data from a string value"""
|
||||
if not sensitive_data:
|
||||
return value
|
||||
|
||||
# Collect all sensitive values, immediately converting old format to new format
|
||||
sensitive_values: dict[str, str] = {}
|
||||
|
||||
# Process all sensitive data entries
|
||||
for key_or_domain, content in sensitive_data.items():
|
||||
if isinstance(content, dict):
|
||||
# Already in new format: {domain: {key: value}}
|
||||
for key, val in content.items():
|
||||
if val: # Skip empty values
|
||||
sensitive_values[key] = val
|
||||
elif content: # Old format: {key: value} - convert to new format internally
|
||||
# We treat this as if it was {'http*://*': {key_or_domain: content}}
|
||||
sensitive_values[key_or_domain] = content
|
||||
|
||||
# If there are no valid sensitive data entries, just return the original value
|
||||
if not sensitive_values:
|
||||
return value
|
||||
|
||||
# Replace all valid sensitive data values with their placeholder tags
|
||||
for key, val in sensitive_values.items():
|
||||
value = value.replace(val, f'<secret>{key}</secret>')
|
||||
|
||||
return value
|
||||
|
||||
def _filter_sensitive_data_from_dict(
|
||||
self, data: dict[str, Any], sensitive_data: dict[str, str | dict[str, str]] | None
|
||||
) -> dict[str, Any]:
|
||||
"""Recursively filter sensitive data from a dictionary"""
|
||||
if not sensitive_data:
|
||||
return data
|
||||
|
||||
filtered_data = {}
|
||||
for key, value in data.items():
|
||||
if isinstance(value, str):
|
||||
filtered_data[key] = self._filter_sensitive_data_from_string(value, sensitive_data)
|
||||
elif isinstance(value, dict):
|
||||
filtered_data[key] = self._filter_sensitive_data_from_dict(value, sensitive_data)
|
||||
elif isinstance(value, list):
|
||||
filtered_data[key] = [
|
||||
self._filter_sensitive_data_from_string(item, sensitive_data)
|
||||
if isinstance(item, str)
|
||||
else self._filter_sensitive_data_from_dict(item, sensitive_data)
|
||||
if isinstance(item, dict)
|
||||
else item
|
||||
for item in value
|
||||
]
|
||||
else:
|
||||
filtered_data[key] = value
|
||||
return filtered_data
|
||||
|
||||
def model_dump(self, sensitive_data: dict[str, str | dict[str, str]] | None = None, **kwargs) -> dict[str, Any]:
|
||||
"""Custom serialization handling circular references and filtering sensitive data"""
|
||||
|
||||
# Handle action serialization
|
||||
model_output_dump = None
|
||||
if self.model_output:
|
||||
action_dump = [action.model_dump(exclude_none=True) for action in self.model_output.action]
|
||||
|
||||
# Filter sensitive data only from input_text action parameters if sensitive_data is provided
|
||||
if sensitive_data:
|
||||
action_dump = [
|
||||
self._filter_sensitive_data_from_dict(action, sensitive_data)
|
||||
if action.get('name') == 'input_text'
|
||||
else action
|
||||
for action in action_dump
|
||||
]
|
||||
|
||||
model_output_dump = {
|
||||
'evaluation_previous_goal': self.model_output.evaluation_previous_goal,
|
||||
'memory': self.model_output.memory,
|
||||
'next_goal': self.model_output.next_goal,
|
||||
'action': action_dump, # This preserves the actual action data
|
||||
}
|
||||
# Only include thinking if it's present
|
||||
if self.model_output.thinking is not None:
|
||||
model_output_dump['thinking'] = self.model_output.thinking
|
||||
|
||||
# Handle result serialization - don't filter ActionResult data
|
||||
# as it should contain meaningful information for the agent
|
||||
result_dump = [r.model_dump(exclude_none=True) for r in self.result]
|
||||
|
||||
return {
|
||||
'model_output': model_output_dump,
|
||||
'result': result_dump,
|
||||
'state': self.state.to_dict(),
|
||||
'metadata': self.metadata.model_dump() if self.metadata else None,
|
||||
}
|
||||
|
||||
|
||||
AgentStructuredOutput = TypeVar('AgentStructuredOutput', bound=BaseModel)
|
||||
|
||||
|
||||
class AgentHistoryList(BaseModel, Generic[AgentStructuredOutput]):
|
||||
"""List of AgentHistory messages, i.e. the history of the agent's actions and thoughts."""
|
||||
|
||||
history: list[AgentHistory]
|
||||
usage: UsageSummary | None = None
|
||||
|
||||
_output_model_schema: type[AgentStructuredOutput] | None = None
|
||||
|
||||
def total_duration_seconds(self) -> float:
|
||||
"""Get total duration of all steps in seconds"""
|
||||
total = 0.0
|
||||
for h in self.history:
|
||||
if h.metadata:
|
||||
total += h.metadata.duration_seconds
|
||||
return total
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""Return the number of history items"""
|
||||
return len(self.history)
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""Representation of the AgentHistoryList object"""
|
||||
return f'AgentHistoryList(all_results={self.action_results()}, all_model_outputs={self.model_actions()})'
|
||||
|
||||
def add_item(self, history_item: AgentHistory) -> None:
|
||||
"""Add a history item to the list"""
|
||||
self.history.append(history_item)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""Representation of the AgentHistoryList object"""
|
||||
return self.__str__()
|
||||
|
||||
def save_to_file(self, filepath: str | Path, sensitive_data: dict[str, str | dict[str, str]] | None = None) -> None:
|
||||
"""Save history to JSON file with proper serialization and optional sensitive data filtering"""
|
||||
try:
|
||||
Path(filepath).parent.mkdir(parents=True, exist_ok=True)
|
||||
data = self.model_dump(sensitive_data=sensitive_data)
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
# def save_as_playwright_script(
|
||||
# self,
|
||||
# output_path: str | Path,
|
||||
# sensitive_data_keys: list[str] | None = None,
|
||||
# browser_config: BrowserConfig | None = None,
|
||||
# context_config: BrowserContextConfig | None = None,
|
||||
# ) -> None:
|
||||
# """
|
||||
# Generates a Playwright script based on the agent's history and saves it to a file.
|
||||
# Args:
|
||||
# output_path: The path where the generated Python script will be saved.
|
||||
# sensitive_data_keys: A list of keys used as placeholders for sensitive data
|
||||
# (e.g., ['username_placeholder', 'password_placeholder']).
|
||||
# These will be loaded from environment variables in the
|
||||
# generated script.
|
||||
# browser_config: Configuration of the original Browser instance.
|
||||
# context_config: Configuration of the original BrowserContext instance.
|
||||
# """
|
||||
# from browser_use.agent.playwright_script_generator import PlaywrightScriptGenerator
|
||||
|
||||
# try:
|
||||
# serialized_history = self.model_dump()['history']
|
||||
# generator = PlaywrightScriptGenerator(serialized_history, sensitive_data_keys, browser_config, context_config)
|
||||
|
||||
# script_content = generator.generate_script_content()
|
||||
# path_obj = Path(output_path)
|
||||
# path_obj.parent.mkdir(parents=True, exist_ok=True)
|
||||
# with open(path_obj, 'w', encoding='utf-8') as f:
|
||||
# f.write(script_content)
|
||||
# except Exception as e:
|
||||
# raise e
|
||||
|
||||
def model_dump(self, **kwargs) -> dict[str, Any]:
|
||||
"""Custom serialization that properly uses AgentHistory's model_dump"""
|
||||
return {
|
||||
'history': [h.model_dump(**kwargs) for h in self.history],
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def load_from_file(cls, filepath: str | Path, output_model: type[AgentOutput]) -> AgentHistoryList:
|
||||
"""Load history from JSON file"""
|
||||
with open(filepath, encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
# loop through history and validate output_model actions to enrich with custom actions
|
||||
for h in data['history']:
|
||||
if h['model_output']:
|
||||
if isinstance(h['model_output'], dict):
|
||||
h['model_output'] = output_model.model_validate(h['model_output'])
|
||||
else:
|
||||
h['model_output'] = None
|
||||
if 'interacted_element' not in h['state']:
|
||||
h['state']['interacted_element'] = None
|
||||
history = cls.model_validate(data)
|
||||
return history
|
||||
|
||||
def last_action(self) -> None | dict:
|
||||
"""Last action in history"""
|
||||
if self.history and self.history[-1].model_output:
|
||||
return self.history[-1].model_output.action[-1].model_dump(exclude_none=True)
|
||||
return None
|
||||
|
||||
def errors(self) -> list[str | None]:
|
||||
"""Get all errors from history, with None for steps without errors"""
|
||||
errors = []
|
||||
for h in self.history:
|
||||
step_errors = [r.error for r in h.result if r.error]
|
||||
|
||||
# each step can have only one error
|
||||
errors.append(step_errors[0] if step_errors else None)
|
||||
return errors
|
||||
|
||||
def final_result(self) -> None | str:
|
||||
"""Final result from history"""
|
||||
if self.history and self.history[-1].result[-1].extracted_content:
|
||||
return self.history[-1].result[-1].extracted_content
|
||||
return None
|
||||
|
||||
def is_done(self) -> bool:
|
||||
"""Check if the agent is done"""
|
||||
if self.history and len(self.history[-1].result) > 0:
|
||||
last_result = self.history[-1].result[-1]
|
||||
return last_result.is_done is True
|
||||
return False
|
||||
|
||||
def is_successful(self) -> bool | None:
|
||||
"""Check if the agent completed successfully - the agent decides in the last step if it was successful or not. None if not done yet."""
|
||||
if self.history and len(self.history[-1].result) > 0:
|
||||
last_result = self.history[-1].result[-1]
|
||||
if last_result.is_done is True:
|
||||
return last_result.success
|
||||
return None
|
||||
|
||||
def has_errors(self) -> bool:
|
||||
"""Check if the agent has any non-None errors"""
|
||||
return any(error is not None for error in self.errors())
|
||||
|
||||
def urls(self) -> list[str | None]:
|
||||
"""Get all unique URLs from history"""
|
||||
return [h.state.url if h.state.url is not None else None for h in self.history]
|
||||
|
||||
def screenshot_paths(self, n_last: int | None = None, return_none_if_not_screenshot: bool = True) -> list[str | None]:
|
||||
"""Get all screenshot paths from history"""
|
||||
if n_last == 0:
|
||||
return []
|
||||
if n_last is None:
|
||||
if return_none_if_not_screenshot:
|
||||
return [h.state.screenshot_path if h.state.screenshot_path is not None else None for h in self.history]
|
||||
else:
|
||||
return [h.state.screenshot_path for h in self.history if h.state.screenshot_path is not None]
|
||||
else:
|
||||
if return_none_if_not_screenshot:
|
||||
return [h.state.screenshot_path if h.state.screenshot_path is not None else None for h in self.history[-n_last:]]
|
||||
else:
|
||||
return [h.state.screenshot_path for h in self.history[-n_last:] if h.state.screenshot_path is not None]
|
||||
|
||||
def screenshots(self, n_last: int | None = None, return_none_if_not_screenshot: bool = True) -> list[str | None]:
|
||||
"""Get all screenshots from history as base64 strings"""
|
||||
if n_last == 0:
|
||||
return []
|
||||
|
||||
history_items = self.history if n_last is None else self.history[-n_last:]
|
||||
screenshots = []
|
||||
|
||||
for item in history_items:
|
||||
screenshot_b64 = item.state.get_screenshot()
|
||||
if screenshot_b64:
|
||||
screenshots.append(screenshot_b64)
|
||||
else:
|
||||
if return_none_if_not_screenshot:
|
||||
screenshots.append(None)
|
||||
# If return_none_if_not_screenshot is False, we skip None values
|
||||
|
||||
return screenshots
|
||||
|
||||
def action_names(self) -> list[str]:
|
||||
"""Get all action names from history"""
|
||||
action_names = []
|
||||
for action in self.model_actions():
|
||||
actions = list(action.keys())
|
||||
if actions:
|
||||
action_names.append(actions[0])
|
||||
return action_names
|
||||
|
||||
def model_thoughts(self) -> list[AgentBrain]:
|
||||
"""Get all thoughts from history"""
|
||||
return [h.model_output.current_state for h in self.history if h.model_output]
|
||||
|
||||
def model_outputs(self) -> list[AgentOutput]:
|
||||
"""Get all model outputs from history"""
|
||||
return [h.model_output for h in self.history if h.model_output]
|
||||
|
||||
# get all actions with params
|
||||
def model_actions(self) -> list[dict]:
|
||||
"""Get all actions from history"""
|
||||
outputs = []
|
||||
|
||||
for h in self.history:
|
||||
if h.model_output:
|
||||
# Guard against None interacted_element before zipping
|
||||
interacted_elements = h.state.interacted_element or [None] * len(h.model_output.action)
|
||||
for action, interacted_element in zip(h.model_output.action, interacted_elements):
|
||||
output = action.model_dump(exclude_none=True)
|
||||
output['interacted_element'] = interacted_element
|
||||
outputs.append(output)
|
||||
return outputs
|
||||
|
||||
def action_history(self) -> list[list[dict]]:
|
||||
"""Get truncated action history with only essential fields"""
|
||||
step_outputs = []
|
||||
|
||||
for h in self.history:
|
||||
step_actions = []
|
||||
if h.model_output:
|
||||
# Guard against None interacted_element before zipping
|
||||
interacted_elements = h.state.interacted_element or [None] * len(h.model_output.action)
|
||||
# Zip actions with interacted elements and results
|
||||
for action, interacted_element, result in zip(h.model_output.action, interacted_elements, h.result):
|
||||
action_output = action.model_dump(exclude_none=True)
|
||||
action_output['interacted_element'] = interacted_element
|
||||
# Only keep long_term_memory from result
|
||||
action_output['result'] = result.long_term_memory if result and result.long_term_memory else None
|
||||
step_actions.append(action_output)
|
||||
step_outputs.append(step_actions)
|
||||
|
||||
return step_outputs
|
||||
|
||||
def action_results(self) -> list[ActionResult]:
|
||||
"""Get all results from history"""
|
||||
results = []
|
||||
for h in self.history:
|
||||
results.extend([r for r in h.result if r])
|
||||
return results
|
||||
|
||||
def extracted_content(self) -> list[str]:
|
||||
"""Get all extracted content from history"""
|
||||
content = []
|
||||
for h in self.history:
|
||||
content.extend([r.extracted_content for r in h.result if r.extracted_content])
|
||||
return content
|
||||
|
||||
def model_actions_filtered(self, include: list[str] | None = None) -> list[dict]:
|
||||
"""Get all model actions from history as JSON"""
|
||||
if include is None:
|
||||
include = []
|
||||
outputs = self.model_actions()
|
||||
result = []
|
||||
for o in outputs:
|
||||
for i in include:
|
||||
if i == list(o.keys())[0]:
|
||||
result.append(o)
|
||||
return result
|
||||
|
||||
def number_of_steps(self) -> int:
|
||||
"""Get the number of steps in the history"""
|
||||
return len(self.history)
|
||||
|
||||
@property
|
||||
def structured_output(self) -> AgentStructuredOutput | None:
|
||||
"""Get the structured output from the history
|
||||
|
||||
Returns:
|
||||
The structured output if both final_result and _output_model_schema are available,
|
||||
otherwise None
|
||||
"""
|
||||
final_result = self.final_result()
|
||||
if final_result is not None and self._output_model_schema is not None:
|
||||
return self._output_model_schema.model_validate_json(final_result)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class AgentError:
|
||||
"""Container for agent error handling"""
|
||||
|
||||
VALIDATION_ERROR = 'Invalid model output format. Please follow the correct schema.'
|
||||
RATE_LIMIT_ERROR = 'Rate limit reached. Waiting before retry.'
|
||||
NO_VALID_ACTION = 'No valid action found'
|
||||
|
||||
@staticmethod
|
||||
def format_error(error: Exception, include_trace: bool = False) -> str:
|
||||
"""Format error message based on error type and optionally include trace"""
|
||||
message = ''
|
||||
if isinstance(error, ValidationError):
|
||||
return f'{AgentError.VALIDATION_ERROR}\nDetails: {str(error)}'
|
||||
if isinstance(error, RateLimitError):
|
||||
return AgentError.RATE_LIMIT_ERROR
|
||||
if include_trace:
|
||||
return f'{str(error)}\nStacktrace:\n{traceback.format_exc()}'
|
||||
return f'{str(error)}'
|
||||
Reference in New Issue
Block a user