ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter9/browser-use-rpa/browser-use/tests/agent_tasks/README.md

@@ -0,0 +1,33 @@
+# Contributing Agent Tasks
+
+Contribute your own agent tasks and we test if the agent solves them for CI testing!
+
+## How to Add a Task
+
+1. Create a new `.yaml` file in this directory (`tests/agent_tasks/`).
+2. Use the following format:
+
+```yaml
+name: My Task Name
+task: Describe the task for the agent to perform
+judge_context:
+  - List criteria for success, one per line
+max_steps: 10
+```
+
+## Guidelines
+- Be specific in your task and criteria.
+- The `judge_context` should list what counts as a successful result.
+- The agent's output will be judged by an LLM using these criteria.
+
+## Running the Tests
+
+To run all agent tasks:
+
+```bash
+pytest tests/ci/test_agent_real_tasks.py
+```
+
+---
+
+Happy contributing! 

chapter9/browser-use-rpa/browser-use/tests/agent_tasks/amazon_laptop.yaml

@@ -0,0 +1,7 @@
+name: Amazon Laptop Search
+task: Go to amazon.com, search for 'laptop', and return the first result
+judge_context:
+  - The agent must navigate to amazon.com
+  - The agent must search for 'laptop'
+  - The agent must return name of the first laptop 
+max_steps: 10

chapter9/browser-use-rpa/browser-use/tests/agent_tasks/browser_use_pip.yaml

@@ -0,0 +1,5 @@
+name: Find pip install command for browser-use
+task: Find the pip installation command for the browser-use repo
+judge_context:
+  - The output must include the command ('pip install browser-use') 
+max_steps: 10

chapter9/browser-use-rpa/browser-use/tests/ci/conftest.py

@@ -0,0 +1,239 @@
+"""
+Pytest configuration for browser-use CI tests.
+
+Sets up environment variables to ensure tests never connect to production services.
+"""
+
+import os
+import socketserver
+import tempfile
+from unittest.mock import AsyncMock
+
+import pytest
+from dotenv import load_dotenv
+from pytest_httpserver import HTTPServer
+
+# Fix for httpserver hanging on shutdown - prevent blocking on socket close
+# This prevents tests from hanging when shutting down HTTP servers
+socketserver.ThreadingMixIn.block_on_close = False
+# Also set daemon threads to prevent hanging
+socketserver.ThreadingMixIn.daemon_threads = True
+
+from browser_use.agent.views import AgentOutput
+from browser_use.llm import BaseChatModel
+from browser_use.llm.views import ChatInvokeCompletion
+from browser_use.tools.service import Tools
+
+# Load environment variables before any imports
+load_dotenv()
+
+
+# Skip LLM API key verification for tests
+os.environ['SKIP_LLM_API_KEY_VERIFICATION'] = 'true'
+
+from bubus import BaseEvent
+
+from browser_use import Agent
+from browser_use.browser import BrowserProfile, BrowserSession
+from browser_use.sync.service import CloudSync
+
+
+@pytest.fixture(autouse=True)
+def setup_test_environment():
+	"""
+	Automatically set up test environment for all tests.
+	"""
+
+	# Create a temporary directory for test config (but not for extensions)
+	config_dir = tempfile.mkdtemp(prefix='browseruse_tests_')
+
+	original_env = {}
+	test_env_vars = {
+		'SKIP_LLM_API_KEY_VERIFICATION': 'true',
+		'ANONYMIZED_TELEMETRY': 'false',
+		'BROWSER_USE_CLOUD_SYNC': 'true',
+		'BROWSER_USE_CLOUD_API_URL': 'http://placeholder-will-be-replaced-by-specific-test-fixtures',
+		'BROWSER_USE_CLOUD_UI_URL': 'http://placeholder-will-be-replaced-by-specific-test-fixtures',
+		# Don't set BROWSER_USE_CONFIG_DIR anymore - let it use the default ~/.config/browseruse
+		# This way extensions will be cached in ~/.config/browseruse/extensions
+	}
+
+	for key, value in test_env_vars.items():
+		original_env[key] = os.environ.get(key)
+		os.environ[key] = value
+
+	yield
+
+	# Restore original environment
+	for key, value in original_env.items():
+		if value is None:
+			os.environ.pop(key, None)
+		else:
+			os.environ[key] = value
+
+
+# not a fixture, mock_llm() provides this in a fixture below, this is a helper so that it can accept args
+def create_mock_llm(actions: list[str] | None = None) -> BaseChatModel:
+	"""Create a mock LLM that returns specified actions or a default done action.
+
+	Args:
+		actions: Optional list of JSON strings representing actions to return in sequence.
+			If not provided, returns a single done action.
+			After all actions are exhausted, returns a done action.
+
+	Returns:
+		Mock LLM that will return the actions in order, or just a done action if no actions provided.
+	"""
+	tools = Tools()
+	ActionModel = tools.registry.create_action_model()
+	AgentOutputWithActions = AgentOutput.type_with_custom_actions(ActionModel)
+
+	llm = AsyncMock(spec=BaseChatModel)
+	llm.model = 'mock-llm'
+	llm._verified_api_keys = True
+
+	# Add missing properties from BaseChatModel protocol
+	llm.provider = 'mock'
+	llm.name = 'mock-llm'
+	llm.model_name = 'mock-llm'  # Ensure this returns a string, not a mock
+
+	# Default done action
+	default_done_action = """
+	{
+		"thinking": "null",
+		"evaluation_previous_goal": "Successfully completed the task",
+		"memory": "Task completed",
+		"next_goal": "Task completed",
+		"action": [
+			{
+				"done": {
+					"text": "Task completed successfully",
+					"success": true
+				}
+			}
+		]
+	}
+	"""
+
+	# Unified logic for both cases
+	action_index = 0
+
+	def get_next_action() -> str:
+		nonlocal action_index
+		if actions is not None and action_index < len(actions):
+			action = actions[action_index]
+			action_index += 1
+			return action
+		else:
+			return default_done_action
+
+	async def mock_ainvoke(*args, **kwargs):
+		# Check if output_format is provided (2nd argument or in kwargs)
+		output_format = None
+		if len(args) >= 2:
+			output_format = args[1]
+		elif 'output_format' in kwargs:
+			output_format = kwargs['output_format']
+
+		action_json = get_next_action()
+
+		if output_format is None:
+			# Return string completion
+			return ChatInvokeCompletion(completion=action_json, usage=None)
+		else:
+			# Parse with provided output_format (could be AgentOutputWithActions or another model)
+			if output_format == AgentOutputWithActions:
+				parsed = AgentOutputWithActions.model_validate_json(action_json)
+			else:
+				# For other output formats, try to parse the JSON with that model
+				parsed = output_format.model_validate_json(action_json)
+			return ChatInvokeCompletion(completion=parsed, usage=None)
+
+	llm.ainvoke.side_effect = mock_ainvoke
+
+	return llm
+
+
+@pytest.fixture(scope='module')
+async def browser_session():
+	"""Create a real browser session for testing"""
+	session = BrowserSession(
+		browser_profile=BrowserProfile(
+			headless=True,
+			user_data_dir=None,  # Use temporary directory
+			keep_alive=True,
+			enable_default_extensions=True,  # Enable extensions during tests
+		)
+	)
+	await session.start()
+	yield session
+	await session.kill()
+	# Ensure event bus is properly stopped
+	await session.event_bus.stop(clear=True, timeout=5)
+
+
+@pytest.fixture(scope='function')
+def cloud_sync(httpserver: HTTPServer):
+	"""
+	Create a CloudSync instance configured for testing.
+
+	This fixture creates a real CloudSync instance and sets up the test environment
+	to use the httpserver URLs.
+	"""
+
+	# Set up test environment
+	test_http_server_url = httpserver.url_for('')
+	os.environ['BROWSER_USE_CLOUD_API_URL'] = test_http_server_url
+	os.environ['BROWSER_USE_CLOUD_UI_URL'] = test_http_server_url
+	os.environ['BROWSER_USE_CLOUD_SYNC'] = 'true'
+
+	# Create CloudSync with test server URL
+	cloud_sync = CloudSync(
+		base_url=test_http_server_url,
+	)
+
+	return cloud_sync
+
+
+@pytest.fixture(scope='function')
+def mock_llm():
+	"""Create a mock LLM that just returns the done action if queried"""
+	return create_mock_llm(actions=None)
+
+
+@pytest.fixture(scope='function')
+def agent_with_cloud(browser_session, mock_llm, cloud_sync):
+	"""Create agent with cloud sync enabled (using real CloudSync)."""
+	agent = Agent(
+		task='Test task',
+		llm=mock_llm,
+		browser_session=browser_session,
+		cloud_sync=cloud_sync,
+	)
+	return agent
+
+
+@pytest.fixture(scope='function')
+def event_collector():
+	"""Helper to collect all events emitted during tests"""
+	events = []
+	event_order = []
+
+	class EventCollector:
+		def __init__(self):
+			self.events = events
+			self.event_order = event_order
+
+		async def collect_event(self, event: BaseEvent):
+			self.events.append(event)
+			self.event_order.append(event.event_type)
+			return 'collected'
+
+		def get_events_by_type(self, event_type: str) -> list[BaseEvent]:
+			return [e for e in self.events if e.event_type == event_type]
+
+		def clear(self):
+			self.events.clear()
+			self.event_order.clear()
+
+	return EventCollector()

chapter9/browser-use-rpa/browser-use/tests/ci/evaluate_tasks.py

@@ -0,0 +1,346 @@
+"""
+Runs all agent tasks in parallel (up to 10 at a time) using separate subprocesses.
+Each task gets its own Python process, preventing browser session interference.
+Does not fail on partial failures (always exits 0).
+"""
+
+import argparse
+import asyncio
+import glob
+import json
+import logging
+import os
+import sys
+import warnings
+
+import aiofiles
+import yaml
+from pydantic import BaseModel
+
+from browser_use import Agent, AgentHistoryList, BrowserProfile, BrowserSession, ChatOpenAI
+from browser_use.llm.messages import UserMessage
+
+# --- CONFIG ---
+MAX_PARALLEL = 10
+TASK_DIR = (
+	sys.argv[1]
+	if len(sys.argv) > 1 and not sys.argv[1].startswith('--')
+	else os.path.join(os.path.dirname(__file__), '../agent_tasks')
+)
+TASK_FILES = glob.glob(os.path.join(TASK_DIR, '*.yaml'))
+
+
+class JudgeResponse(BaseModel):
+	success: bool
+	explanation: str
+
+
+async def run_single_task(task_file):
+	"""Run a single task in the current process (called by subprocess)"""
+	try:
+		print(f'[DEBUG] Starting task: {os.path.basename(task_file)}', file=sys.stderr)
+
+		# Suppress all logging in subprocess to avoid interfering with JSON output
+		logging.getLogger().setLevel(logging.CRITICAL)
+		for logger_name in ['browser_use', 'telemetry', 'message_manager']:
+			logging.getLogger(logger_name).setLevel(logging.CRITICAL)
+		warnings.filterwarnings('ignore')
+
+		print('[DEBUG] Loading task file...', file=sys.stderr)
+		async with aiofiles.open(task_file, 'r') as f:
+			content = await f.read()
+		task_data = yaml.safe_load(content)
+		task = task_data['task']
+		judge_context = task_data.get('judge_context', ['The agent must solve the task'])
+		max_steps = task_data.get('max_steps', 15)
+
+		print(f'[DEBUG] Task: {task[:100]}...', file=sys.stderr)
+		print(f'[DEBUG] Max steps: {max_steps}', file=sys.stderr)
+
+		agent_llm = ChatOpenAI(model='gpt-4.1-mini')
+		judge_llm = ChatOpenAI(model='gpt-4.1-mini')
+		print('[DEBUG] LLMs initialized', file=sys.stderr)
+
+		# Each subprocess gets its own profile and session
+		print('[DEBUG] Creating browser session...', file=sys.stderr)
+		profile = BrowserProfile(
+			headless=True,
+			user_data_dir=None,
+			chromium_sandbox=False,  # Disable sandbox for CI environment (GitHub Actions)
+		)
+		session = BrowserSession(browser_profile=profile)
+		print('[DEBUG] Browser session created', file=sys.stderr)
+
+		# Test if browser is working
+		try:
+			await session.start()
+			from browser_use.browser.events import NavigateToUrlEvent
+
+			event = session.event_bus.dispatch(NavigateToUrlEvent(url='https://httpbin.org/get', new_tab=True))
+			await event
+			print('[DEBUG] Browser test: navigation successful', file=sys.stderr)
+			title = await session.get_current_page_title()
+			print(f"[DEBUG] Browser test: got title '{title}'", file=sys.stderr)
+		except Exception as browser_error:
+			print(f'[DEBUG] Browser test failed: {str(browser_error)}', file=sys.stderr)
+			print(
+				f'[DEBUG] Browser error type: {type(browser_error).__name__}',
+				file=sys.stderr,
+			)
+
+		print('[DEBUG] Starting agent execution...', file=sys.stderr)
+		agent = Agent(task=task, llm=agent_llm, browser_session=session)
+
+		try:
+			history: AgentHistoryList = await agent.run(max_steps=max_steps)
+			print('[DEBUG] Agent.run() returned successfully', file=sys.stderr)
+		except Exception as agent_error:
+			print(
+				f'[DEBUG] Agent.run() failed with error: {str(agent_error)}',
+				file=sys.stderr,
+			)
+			print(f'[DEBUG] Error type: {type(agent_error).__name__}', file=sys.stderr)
+			# Re-raise to be caught by outer try-catch
+			raise agent_error
+
+		agent_output = history.final_result() or ''
+		print('[DEBUG] Agent execution completed', file=sys.stderr)
+
+		# Test if LLM is working by making a simple call
+		try:
+			response = await agent_llm.ainvoke([UserMessage(content="Say 'test'")])
+			print(
+				f'[DEBUG] LLM test call successful: {response.completion[:50]}',
+				file=sys.stderr,
+			)
+		except Exception as llm_error:
+			print(f'[DEBUG] LLM test call failed: {str(llm_error)}', file=sys.stderr)
+
+		# Debug: capture more details about the agent execution
+		total_steps = len(history.history) if hasattr(history, 'history') else 0
+		last_action = history.history[-1] if hasattr(history, 'history') and history.history else None
+		debug_info = f'Steps: {total_steps}, Final result length: {len(agent_output)}'
+		if last_action:
+			debug_info += f', Last action: {type(last_action).__name__}'
+
+		# Log to stderr so it shows up in GitHub Actions (won't interfere with JSON output to stdout)
+		print(f'[DEBUG] Task {os.path.basename(task_file)}: {debug_info}', file=sys.stderr)
+		if agent_output:
+			print(
+				f'[DEBUG] Agent output preview: {agent_output[:200]}...',
+				file=sys.stderr,
+			)
+		else:
+			print('[DEBUG] Agent produced no output!', file=sys.stderr)
+
+		criteria = '\n- '.join(judge_context)
+		judge_prompt = f"""
+You are a evaluator of a browser agent task inside a ci/cd pipeline. Here was the agent's task:
+{task}
+
+Here is the agent's output:
+{agent_output if agent_output else '[No output provided]'}
+
+Debug info: {debug_info}
+
+Criteria for success:
+- {criteria}
+
+Reply in JSON with keys: success (true/false), explanation (string).
+If the agent provided no output, explain what might have gone wrong.
+"""
+		response = await judge_llm.ainvoke([UserMessage(content=judge_prompt)], output_format=JudgeResponse)
+		judge_response = response.completion
+
+		result = {
+			'file': os.path.basename(task_file),
+			'success': judge_response.success,
+			'explanation': judge_response.explanation,
+		}
+
+		# Clean up session before returning
+		await session.kill()
+
+		return result
+
+	except Exception as e:
+		# Ensure session cleanup even on error
+		try:
+			await session.kill()
+		except Exception:
+			pass
+
+		return {
+			'file': os.path.basename(task_file),
+			'success': False,
+			'explanation': f'Task failed with error: {str(e)}',
+		}
+
+
+async def run_task_subprocess(task_file, semaphore):
+	"""Run a task in a separate subprocess"""
+	async with semaphore:
+		try:
+			# Set environment to reduce noise in subprocess
+			env = os.environ.copy()
+			env['PYTHONPATH'] = os.pathsep.join(sys.path)
+
+			proc = await asyncio.create_subprocess_exec(
+				sys.executable,
+				__file__,
+				'--task',
+				task_file,
+				stdout=asyncio.subprocess.PIPE,
+				stderr=asyncio.subprocess.PIPE,
+				env=env,
+			)
+			stdout, stderr = await proc.communicate()
+
+			if proc.returncode == 0:
+				try:
+					# Parse JSON result from subprocess
+					stdout_text = stdout.decode().strip()
+					stderr_text = stderr.decode().strip()
+
+					# Display subprocess debug logs
+					if stderr_text:
+						print(f'[SUBPROCESS {os.path.basename(task_file)}] Debug output:')
+						for line in stderr_text.split('\n'):
+							if line.strip():
+								print(f'  {line}')
+
+					# Find the JSON line (should be the last line that starts with {)
+					lines = stdout_text.split('\n')
+					json_line = None
+					for line in reversed(lines):
+						line = line.strip()
+						if line.startswith('{') and line.endswith('}'):
+							json_line = line
+							break
+
+					if json_line:
+						result = json.loads(json_line)
+						print(f'[PARENT] Task {os.path.basename(task_file)} completed: {result["success"]}')
+					else:
+						raise ValueError(f'No JSON found in output: {stdout_text}')
+
+				except (json.JSONDecodeError, ValueError) as e:
+					result = {
+						'file': os.path.basename(task_file),
+						'success': False,
+						'explanation': f'Failed to parse subprocess result: {str(e)[:100]}',
+					}
+					print(f'[PARENT] Task {os.path.basename(task_file)} failed to parse: {str(e)}')
+					print(f'[PARENT] Full stdout was: {stdout.decode()[:500]}')
+			else:
+				stderr_text = stderr.decode().strip()
+				result = {
+					'file': os.path.basename(task_file),
+					'success': False,
+					'explanation': f'Subprocess failed (code {proc.returncode}): {stderr_text[:200]}',
+				}
+				print(f'[PARENT] Task {os.path.basename(task_file)} subprocess failed with code {proc.returncode}')
+				if stderr_text:
+					print(f'[PARENT] stderr: {stderr_text[:1000]}')
+				stdout_text = stdout.decode().strip()
+				if stdout_text:
+					print(f'[PARENT] stdout: {stdout_text[:1000]}')
+		except Exception as e:
+			result = {
+				'file': os.path.basename(task_file),
+				'success': False,
+				'explanation': f'Failed to start subprocess: {str(e)}',
+			}
+			print(f'[PARENT] Failed to start subprocess for {os.path.basename(task_file)}: {str(e)}')
+
+		return result
+
+
+async def main():
+	"""Run all tasks in parallel using subprocesses"""
+	semaphore = asyncio.Semaphore(MAX_PARALLEL)
+
+	print(f'Found task files: {TASK_FILES}')
+
+	if not TASK_FILES:
+		print('No task files found!')
+		return 0, 0
+
+	# Run all tasks in parallel subprocesses
+	tasks = [run_task_subprocess(task_file, semaphore) for task_file in TASK_FILES]
+	results = await asyncio.gather(*tasks)
+
+	passed = sum(1 for r in results if r['success'])
+	total = len(results)
+
+	print('\n' + '=' * 60)
+	print(f'{"RESULTS":^60}\n')
+
+	# Prepare table data
+	headers = ['Task', 'Success', 'Reason']
+	rows = []
+	for r in results:
+		status = '✅' if r['success'] else '❌'
+		rows.append([r['file'], status, r['explanation']])
+
+	# Calculate column widths
+	col_widths = [max(len(str(row[i])) for row in ([headers] + rows)) for i in range(3)]
+
+	# Print header
+	header_row = ' | '.join(headers[i].ljust(col_widths[i]) for i in range(3))
+	print(header_row)
+	print('-+-'.join('-' * w for w in col_widths))
+
+	# Print rows
+	for row in rows:
+		print(' | '.join(str(row[i]).ljust(col_widths[i]) for i in range(3)))
+
+	print('\n' + '=' * 60)
+	print(f'\n{"SCORE":^60}')
+	print(f'\n{"=" * 60}\n')
+	print(f'\n{"*" * 10}  {passed}/{total} PASSED  {"*" * 10}\n')
+	print('=' * 60 + '\n')
+
+	# Output results for GitHub Actions
+	print(f'PASSED={passed}')
+	print(f'TOTAL={total}')
+
+	# Output detailed results as JSON for GitHub Actions
+	detailed_results = []
+	for r in results:
+		detailed_results.append(
+			{
+				'task': r['file'].replace('.yaml', ''),
+				'success': r['success'],
+				'reason': r['explanation'],
+			}
+		)
+
+	print('DETAILED_RESULTS=' + json.dumps(detailed_results))
+
+	return passed, total
+
+
+if __name__ == '__main__':
+	parser = argparse.ArgumentParser()
+	parser.add_argument('--task', type=str, help='Path to a single task YAML file (for subprocess mode)')
+	args = parser.parse_args()
+
+	if args.task:
+		# Subprocess mode: run a single task and output ONLY JSON
+		try:
+			result = asyncio.run(run_single_task(args.task))
+			# Output ONLY the JSON result, nothing else
+			print(json.dumps(result))
+		except Exception as e:
+			# Even on critical failure, output valid JSON
+			error_result = {
+				'file': os.path.basename(args.task),
+				'success': False,
+				'explanation': f'Critical subprocess error: {str(e)}',
+			}
+			print(json.dumps(error_result))
+	else:
+		# Parent process mode: run all tasks in parallel subprocesses
+		passed, total = asyncio.run(main())
+		# Results already printed by main() function

chapter9/browser-use-rpa/browser-use/tests/ci/test_agent_sensitive_data.py

@@ -0,0 +1,271 @@
+import pytest
+from pydantic import BaseModel, Field
+
+from browser_use.agent.message_manager.service import MessageManager
+from browser_use.agent.views import MessageManagerState
+from browser_use.filesystem.file_system import FileSystem
+from browser_use.llm import SystemMessage, UserMessage
+from browser_use.llm.messages import ContentPartTextParam
+from browser_use.tools.registry.service import Registry
+from browser_use.utils import is_new_tab_page, match_url_with_domain_pattern
+
+
+class SensitiveParams(BaseModel):
+	"""Test parameter model for sensitive data testing."""
+
+	text: str = Field(description='Text with sensitive data placeholders')
+
+
+@pytest.fixture
+def registry():
+	return Registry()
+
+
+@pytest.fixture
+def message_manager():
+	import os
+	import tempfile
+	import uuid
+
+	base_tmp = tempfile.gettempdir()  # e.g., /tmp on Unix
+	file_system_path = os.path.join(base_tmp, str(uuid.uuid4()))
+	return MessageManager(
+		task='Test task',
+		system_message=SystemMessage(content='System message'),
+		state=MessageManagerState(),
+		file_system=FileSystem(file_system_path),
+	)
+
+
+def test_replace_sensitive_data_with_missing_keys(registry, caplog):
+	"""Test that _replace_sensitive_data handles missing keys gracefully"""
+	# Create a simple Pydantic model with sensitive data placeholders
+	params = SensitiveParams(text='Please enter <secret>username</secret> and <secret>password</secret>')
+
+	# Case 1: All keys present - both placeholders should be replaced
+	sensitive_data = {'username': 'user123', 'password': 'pass456'}
+	result = registry._replace_sensitive_data(params, sensitive_data)
+	assert result.text == 'Please enter user123 and pass456'
+	assert '<secret>' not in result.text  # No secret tags should remain
+
+	# Case 2: One key missing - only available key should be replaced
+	sensitive_data = {'username': 'user123'}  # password is missing
+	result = registry._replace_sensitive_data(params, sensitive_data)
+	assert result.text == 'Please enter user123 and <secret>password</secret>'
+	assert 'user123' in result.text
+	assert '<secret>password</secret>' in result.text  # Missing key's tag remains
+
+	# Case 3: Multiple keys missing - all tags should be preserved
+	sensitive_data = {}  # both keys missing
+	result = registry._replace_sensitive_data(params, sensitive_data)
+	assert result.text == 'Please enter <secret>username</secret> and <secret>password</secret>'
+	assert '<secret>username</secret>' in result.text
+	assert '<secret>password</secret>' in result.text
+
+	# Case 4: One key empty - empty values are treated as missing
+	sensitive_data = {'username': 'user123', 'password': ''}
+	result = registry._replace_sensitive_data(params, sensitive_data)
+	assert result.text == 'Please enter user123 and <secret>password</secret>'
+	assert 'user123' in result.text
+	assert '<secret>password</secret>' in result.text  # Empty value's tag remains
+
+
+def test_simple_domain_specific_sensitive_data(registry, caplog):
+	"""Test the basic functionality of domain-specific sensitive data replacement"""
+	# Create a simple Pydantic model with sensitive data placeholders
+	params = SensitiveParams(text='Please enter <secret>username</secret> and <secret>password</secret>')
+
+	# Simple test with directly instantiable values
+	sensitive_data = {
+		'example.com': {'username': 'example_user'},
+		'other_data': 'non_secret_value',  # Old format mixed with new
+	}
+
+	# Without a URL, domain-specific secrets should NOT be exposed
+	result = registry._replace_sensitive_data(params, sensitive_data)
+	assert result.text == 'Please enter <secret>username</secret> and <secret>password</secret>'
+	assert '<secret>username</secret>' in result.text  # Should NOT be replaced without URL
+	assert '<secret>password</secret>' in result.text  # Password is missing in sensitive_data
+	assert 'example_user' not in result.text  # Domain-specific value should not appear
+
+	# Test with a matching URL - domain-specific secrets should be exposed
+	result = registry._replace_sensitive_data(params, sensitive_data, 'https://example.com/login')
+	assert result.text == 'Please enter example_user and <secret>password</secret>'
+	assert 'example_user' in result.text  # Should be replaced with matching URL
+	assert '<secret>password</secret>' in result.text  # Password is still missing
+	assert '<secret>username</secret>' not in result.text  # Username tag should be replaced
+
+
+def test_match_url_with_domain_pattern():
+	"""Test that the domain pattern matching utility works correctly"""
+
+	# Test exact domain matches
+	assert match_url_with_domain_pattern('https://example.com', 'example.com') is True
+	assert match_url_with_domain_pattern('http://example.com', 'example.com') is False  # Default scheme is now https
+	assert match_url_with_domain_pattern('https://google.com', 'example.com') is False
+
+	# Test subdomain pattern matches
+	assert match_url_with_domain_pattern('https://sub.example.com', '*.example.com') is True
+	assert match_url_with_domain_pattern('https://example.com', '*.example.com') is True  # Base domain should match too
+	assert match_url_with_domain_pattern('https://sub.sub.example.com', '*.example.com') is True
+	assert match_url_with_domain_pattern('https://example.org', '*.example.com') is False
+
+	# Test protocol pattern matches
+	assert match_url_with_domain_pattern('https://example.com', 'http*://example.com') is True
+	assert match_url_with_domain_pattern('http://example.com', 'http*://example.com') is True
+	assert match_url_with_domain_pattern('ftp://example.com', 'http*://example.com') is False
+
+	# Test explicit http protocol
+	assert match_url_with_domain_pattern('http://example.com', 'http://example.com') is True
+	assert match_url_with_domain_pattern('https://example.com', 'http://example.com') is False
+
+	# Test Chrome extension pattern
+	assert match_url_with_domain_pattern('chrome-extension://abcdefghijkl', 'chrome-extension://*') is True
+	assert match_url_with_domain_pattern('chrome-extension://mnopqrstuvwx', 'chrome-extension://abcdefghijkl') is False
+
+	# Test new tab page handling
+	assert match_url_with_domain_pattern('about:blank', 'example.com') is False
+	assert match_url_with_domain_pattern('about:blank', '*://*') is False
+	assert match_url_with_domain_pattern('chrome://new-tab-page/', 'example.com') is False
+	assert match_url_with_domain_pattern('chrome://new-tab-page/', '*://*') is False
+	assert match_url_with_domain_pattern('chrome://new-tab-page', 'example.com') is False
+	assert match_url_with_domain_pattern('chrome://new-tab-page', '*://*') is False
+
+
+def test_unsafe_domain_patterns():
+	"""Test that unsafe domain patterns are rejected"""
+
+	# These are unsafe patterns that could match too many domains
+	assert match_url_with_domain_pattern('https://evil.com', '*google.com') is False
+	assert match_url_with_domain_pattern('https://google.com.evil.com', '*.*.com') is False
+	assert match_url_with_domain_pattern('https://google.com', '**google.com') is False
+	assert match_url_with_domain_pattern('https://google.com', 'g*e.com') is False
+	assert match_url_with_domain_pattern('https://google.com', '*com*') is False
+
+	# Test with patterns that have multiple asterisks in different positions
+	assert match_url_with_domain_pattern('https://subdomain.example.com', '*domain*example*') is False
+	assert match_url_with_domain_pattern('https://sub.domain.example.com', '*.*.example.com') is False
+
+	# Test patterns with wildcards in TLD part
+	assert match_url_with_domain_pattern('https://example.com', 'example.*') is False
+	assert match_url_with_domain_pattern('https://example.org', 'example.*') is False
+
+
+def test_malformed_urls_and_patterns():
+	"""Test handling of malformed URLs or patterns"""
+
+	# Malformed URLs
+	assert match_url_with_domain_pattern('not-a-url', 'example.com') is False
+	assert match_url_with_domain_pattern('http://', 'example.com') is False
+	assert match_url_with_domain_pattern('https://', 'example.com') is False
+	assert match_url_with_domain_pattern('ftp:/example.com', 'example.com') is False  # Missing slash
+
+	# Empty URLs or patterns
+	assert match_url_with_domain_pattern('', 'example.com') is False
+	assert match_url_with_domain_pattern('https://example.com', '') is False
+
+	# URLs with no hostname
+	assert match_url_with_domain_pattern('file:///path/to/file.txt', 'example.com') is False
+
+	# Invalid pattern formats
+	assert match_url_with_domain_pattern('https://example.com', '..example.com') is False
+	assert match_url_with_domain_pattern('https://example.com', '.*.example.com') is False
+	assert match_url_with_domain_pattern('https://example.com', '**') is False
+
+	# Nested URL attacks in path, query or fragments
+	assert match_url_with_domain_pattern('https://example.com/redirect?url=https://evil.com', 'example.com') is True
+	assert match_url_with_domain_pattern('https://example.com/path/https://evil.com', 'example.com') is True
+	assert match_url_with_domain_pattern('https://example.com#https://evil.com', 'example.com') is True
+	# These should match example.com, not evil.com since urlparse extracts the hostname correctly
+
+	# Complex URL obfuscation attempts
+	assert match_url_with_domain_pattern('https://example.com/path?next=//evil.com/attack', 'example.com') is True
+	assert match_url_with_domain_pattern('https://example.com@evil.com', 'example.com') is False
+	assert match_url_with_domain_pattern('https://evil.com?example.com', 'example.com') is False
+	assert match_url_with_domain_pattern('https://user:example.com@evil.com', 'example.com') is False
+	# urlparse correctly identifies evil.com as the hostname in these cases
+
+
+def test_url_components():
+	"""Test handling of URL components like credentials, ports, fragments, etc."""
+
+	# URLs with credentials (username:password@)
+	assert match_url_with_domain_pattern('https://user:pass@example.com', 'example.com') is True
+	assert match_url_with_domain_pattern('https://user:pass@example.com', '*.example.com') is True
+
+	# URLs with ports
+	assert match_url_with_domain_pattern('https://example.com:4242', 'example.com') is True
+	assert match_url_with_domain_pattern('https://example.com:4242', 'example.com:4242') is True  # Port is stripped from pattern
+
+	# URLs with paths
+	assert match_url_with_domain_pattern('https://example.com/path/to/page', 'example.com') is True
+	assert (
+		match_url_with_domain_pattern('https://example.com/path/to/page', 'example.com/path') is False
+	)  # Paths in patterns are not supported
+
+	# URLs with query parameters
+	assert match_url_with_domain_pattern('https://example.com?param=value', 'example.com') is True
+
+	# URLs with fragments
+	assert match_url_with_domain_pattern('https://example.com#section', 'example.com') is True
+
+	# URLs with all components
+	assert match_url_with_domain_pattern('https://user:pass@example.com:4242/path?query=val#fragment', 'example.com') is True
+
+
+def test_filter_sensitive_data(message_manager):
+	"""Test that _filter_sensitive_data handles all sensitive data scenarios correctly"""
+	# Set up a message with sensitive information
+	message = UserMessage(content='My username is admin and password is secret123')
+
+	# Case 1: No sensitive data provided
+	message_manager.sensitive_data = None
+	result = message_manager._filter_sensitive_data(message)
+	assert result.content == 'My username is admin and password is secret123'
+
+	# Case 2: All sensitive data is properly replaced
+	message_manager.sensitive_data = {'username': 'admin', 'password': 'secret123'}
+	result = message_manager._filter_sensitive_data(message)
+	assert '<secret>username</secret>' in result.content
+	assert '<secret>password</secret>' in result.content
+
+	# Case 3: Make sure it works with nested content
+	nested_message = UserMessage(content=[ContentPartTextParam(text='My username is admin and password is secret123')])
+	result = message_manager._filter_sensitive_data(nested_message)
+	assert '<secret>username</secret>' in result.content[0].text
+	assert '<secret>password</secret>' in result.content[0].text
+
+	# Case 4: Test with empty values
+	message_manager.sensitive_data = {'username': 'admin', 'password': ''}
+	result = message_manager._filter_sensitive_data(message)
+	assert '<secret>username</secret>' in result.content
+	# Only username should be replaced since password is empty
+
+	# Case 5: Test with domain-specific sensitive data format
+	message_manager.sensitive_data = {
+		'example.com': {'username': 'admin', 'password': 'secret123'},
+		'google.com': {'email': 'user@example.com', 'password': 'google_pass'},
+	}
+	# Update the message to include the values we're going to test
+	message = UserMessage(content='My username is admin, email is user@example.com and password is secret123 or google_pass')
+	result = message_manager._filter_sensitive_data(message)
+	# All sensitive values should be replaced regardless of domain
+	assert '<secret>username</secret>' in result.content
+	assert '<secret>password</secret>' in result.content
+	assert '<secret>email</secret>' in result.content
+
+
+def test_is_new_tab_page():
+	"""Test is_new_tab_page function"""
+	# Test about:blank
+	assert is_new_tab_page('about:blank') is True
+
+	# Test chrome://new-tab-page variations
+	assert is_new_tab_page('chrome://new-tab-page/') is True
+	assert is_new_tab_page('chrome://new-tab-page') is True
+
+	# Test regular URLs
+	assert is_new_tab_page('https://example.com') is False
+	assert is_new_tab_page('http://google.com') is False
+	assert is_new_tab_page('') is False
+	assert is_new_tab_page('chrome://settings') is False

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_event_GetDropdownOptionsEvent.py

@@ -0,0 +1,670 @@
+"""Test GetDropdownOptionsEvent and SelectDropdownOptionEvent functionality.
+
+This file consolidates all tests related to dropdown functionality including:
+- Native <select> dropdowns
+- ARIA role="menu" dropdowns
+- Custom dropdown implementations
+"""
+
+import pytest
+from pytest_httpserver import HTTPServer
+
+from browser_use.agent.views import ActionModel, ActionResult
+from browser_use.browser import BrowserSession
+from browser_use.browser.events import GetDropdownOptionsEvent, NavigationCompleteEvent, SelectDropdownOptionEvent
+from browser_use.browser.profile import BrowserProfile
+from browser_use.tools.service import Tools
+from browser_use.tools.views import GoToUrlAction
+
+
+@pytest.fixture(scope='session')
+def http_server():
+	"""Create and provide a test HTTP server that serves static content."""
+	server = HTTPServer()
+	server.start()
+
+	# Add route for native dropdown test page
+	server.expect_request('/native-dropdown').respond_with_data(
+		"""
+		<!DOCTYPE html>
+		<html>
+		<head>
+			<title>Native Dropdown Test</title>
+		</head>
+		<body>
+			<h1>Native Dropdown Test</h1>
+			<select id="test-dropdown" name="test-dropdown">
+				<option value="">Please select</option>
+				<option value="option1">First Option</option>
+				<option value="option2">Second Option</option>
+				<option value="option3">Third Option</option>
+			</select>
+			<div id="result">No selection made</div>
+			<script>
+				document.getElementById('test-dropdown').addEventListener('change', function(e) {
+					document.getElementById('result').textContent = 'Selected: ' + e.target.options[e.target.selectedIndex].text;
+				});
+			</script>
+		</body>
+		</html>
+		""",
+		content_type='text/html',
+	)
+
+	# Add route for ARIA menu test page
+	server.expect_request('/aria-menu').respond_with_data(
+		"""
+		<!DOCTYPE html>
+		<html>
+		<head>
+			<title>ARIA Menu Test</title>
+			<style>
+				.menu {
+					list-style: none;
+					padding: 0;
+					margin: 0;
+					border: 1px solid #ccc;
+					background: white;
+					width: 200px;
+				}
+				.menu-item {
+					padding: 10px 20px;
+					border-bottom: 1px solid #eee;
+				}
+				.menu-item:hover {
+					background: #f0f0f0;
+				}
+				.menu-item-anchor {
+					text-decoration: none;
+					color: #333;
+					display: block;
+				}
+				#result {
+					margin-top: 20px;
+					padding: 10px;
+					border: 1px solid #ddd;
+					min-height: 20px;
+				}
+			</style>
+		</head>
+		<body>
+			<h1>ARIA Menu Test</h1>
+			<p>This menu uses ARIA roles instead of native select elements</p>
+			
+			<ul class="menu menu-format-standard menu-regular" role="menu" id="pyNavigation1752753375773" style="display: block;">
+				<li class="menu-item menu-item-enabled" role="presentation">
+					<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+						<span class="menu-item-title-wrap"><span class="menu-item-title">Filter</span></span>
+					</a>
+				</li>
+				<li class="menu-item menu-item-enabled" role="presentation" id="menu-item-$PpyNavigation1752753375773$ppyElements$l2">
+					<a href="#" onclick="pd(event);" class="menu-item-anchor menu-item-expand" tabindex="0" role="menuitem" aria-haspopup="true">
+						<span class="menu-item-title-wrap"><span class="menu-item-title">Sort</span></span>
+					</a>
+					<div class="menu-panel-wrapper">
+						<ul class="menu menu-format-standard menu-regular" role="menu" id="$PpyNavigation1752753375773$ppyElements$l2">
+							<li class="menu-item menu-item-enabled" role="presentation">
+								<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+									<span class="menu-item-title-wrap"><span class="menu-item-title">Lowest to highest</span></span>
+								</a>
+							</li>
+							<li class="menu-item menu-item-enabled" role="presentation">
+								<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+									<span class="menu-item-title-wrap"><span class="menu-item-title">Highest to lowest</span></span>
+								</a>
+							</li>
+						</ul>
+					</div>
+				</li>
+				<li class="menu-item menu-item-enabled" role="presentation">
+					<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+						<span class="menu-item-title-wrap"><span class="menu-item-title">Appearance</span></span>
+					</a>
+				</li>
+				<li class="menu-item menu-item-enabled" role="presentation">
+					<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+						<span class="menu-item-title-wrap"><span class="menu-item-title">Summarize</span></span>
+					</a>
+				</li>
+				<li class="menu-item menu-item-enabled" role="presentation">
+					<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+						<span class="menu-item-title-wrap"><span class="menu-item-title">Delete</span></span>
+					</a>
+				</li>
+			</ul>
+			
+			<div id="result">Click an option to see the result</div>
+			
+			<script>
+				// Mock the pd function that prevents default
+				function pd(event) {
+					event.preventDefault();
+					const text = event.target.closest('[role="menuitem"]').textContent.trim();
+					document.getElementById('result').textContent = 'Clicked: ' + text;
+				}
+			</script>
+		</body>
+		</html>
+		""",
+		content_type='text/html',
+	)
+
+	# Add route for custom dropdown test page
+	server.expect_request('/custom-dropdown').respond_with_data(
+		"""
+		<!DOCTYPE html>
+		<html>
+		<head>
+			<title>Custom Dropdown Test</title>
+			<style>
+				.dropdown {
+					position: relative;
+					display: inline-block;
+					width: 200px;
+				}
+				.dropdown-button {
+					padding: 10px;
+					border: 1px solid #ccc;
+					background: white;
+					cursor: pointer;
+					width: 100%;
+				}
+				.dropdown-menu {
+					position: absolute;
+					top: 100%;
+					left: 0;
+					right: 0;
+					border: 1px solid #ccc;
+					background: white;
+					display: block;
+					z-index: 1000;
+				}
+				.dropdown-menu.hidden {
+					display: none;
+				}
+				.dropdown .item {
+					padding: 10px;
+					cursor: pointer;
+				}
+				.dropdown .item:hover {
+					background: #f0f0f0;
+				}
+				.dropdown .item.selected {
+					background: #e0e0e0;
+				}
+				#result {
+					margin-top: 20px;
+					padding: 10px;
+					border: 1px solid #ddd;
+				}
+			</style>
+		</head>
+		<body>
+			<h1>Custom Dropdown Test</h1>
+			<p>This is a custom dropdown implementation (like Semantic UI)</p>
+			
+			<div class="dropdown ui" id="custom-dropdown">
+				<div class="dropdown-button" onclick="toggleDropdown()">
+					<span id="selected-text">Choose an option</span>
+				</div>
+				<div class="dropdown-menu" id="dropdown-menu">
+					<div class="item" data-value="red" onclick="selectOption('Red', 'red')">Red</div>
+					<div class="item" data-value="green" onclick="selectOption('Green', 'green')">Green</div>
+					<div class="item" data-value="blue" onclick="selectOption('Blue', 'blue')">Blue</div>
+					<div class="item" data-value="yellow" onclick="selectOption('Yellow', 'yellow')">Yellow</div>
+				</div>
+			</div>
+			
+			<div id="result">No selection made</div>
+			
+			<script>
+				function toggleDropdown() {
+					const menu = document.getElementById('dropdown-menu');
+					menu.classList.toggle('hidden');
+				}
+				
+				function selectOption(text, value) {
+					document.getElementById('selected-text').textContent = text;
+					document.getElementById('result').textContent = 'Selected: ' + text + ' (value: ' + value + ')';
+					// Mark as selected
+					document.querySelectorAll('.item').forEach(item => item.classList.remove('selected'));
+					event.target.classList.add('selected');
+					// Close dropdown
+					document.getElementById('dropdown-menu').classList.add('hidden');
+				}
+			</script>
+		</body>
+		</html>
+		""",
+		content_type='text/html',
+	)
+
+	yield server
+	server.stop()
+
+
+@pytest.fixture(scope='session')
+def base_url(http_server):
+	"""Return the base URL for the test HTTP server."""
+	return f'http://{http_server.host}:{http_server.port}'
+
+
+@pytest.fixture(scope='module')
+async def browser_session():
+	"""Create and provide a Browser instance with security disabled."""
+	browser_session = BrowserSession(
+		browser_profile=BrowserProfile(
+			headless=True,
+			user_data_dir=None,
+			keep_alive=True,
+			chromium_sandbox=False,  # Disable sandbox for CI environment
+		)
+	)
+	await browser_session.start()
+	yield browser_session
+	await browser_session.kill()
+
+
+@pytest.fixture(scope='function')
+def tools():
+	"""Create and provide a Tools instance."""
+	return Tools()
+
+
+class TestGetDropdownOptionsEvent:
+	"""Test GetDropdownOptionsEvent functionality for various dropdown types."""
+
+	@pytest.mark.skip(reason='Dropdown text assertion issue - test expects specific text format')
+	async def test_native_select_dropdown(self, tools, browser_session: BrowserSession, base_url):
+		"""Test get_dropdown_options with native HTML select element."""
+		# Navigate to the native dropdown test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/native-dropdown', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+
+		# Initialize the DOM state to populate the selector map
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Get the selector map and find the select element
+		selector_map = await browser_session.get_selector_map()
+		dropdown_index = None
+		for idx, element in selector_map.items():
+			if element.tag_name.lower() == 'select' and element.attributes.get('id') == 'test-dropdown':
+				dropdown_index = idx
+				break
+
+		assert dropdown_index is not None, (
+			f'Could not find select element in selector map. Available elements: {[f"{idx}: {element.tag_name}" for idx, element in selector_map.items()]}'
+		)
+
+		# Test via tools action
+		class GetDropdownOptionsModel(ActionModel):
+			get_dropdown_options: dict[str, int]
+
+		result = await tools.act(
+			action=GetDropdownOptionsModel(get_dropdown_options={'index': dropdown_index}),
+			browser_session=browser_session,
+		)
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+
+		# Verify all expected options are present
+		expected_options = ['Please select', 'First Option', 'Second Option', 'Third Option']
+		for option in expected_options:
+			assert option in result.extracted_content, f"Option '{option}' not found in result content"
+
+		# Verify instruction is included
+		assert 'Use the exact text string' in result.extracted_content and 'select_dropdown_option' in result.extracted_content
+
+		# Also test direct event dispatch
+		node = await browser_session.get_element_by_index(dropdown_index)
+		assert node is not None
+		event = browser_session.event_bus.dispatch(GetDropdownOptionsEvent(node=node))
+		dropdown_data = await event.event_result(timeout=3.0)
+
+		assert dropdown_data is not None
+		assert 'options' in dropdown_data
+		assert 'type' in dropdown_data
+		assert dropdown_data['type'] == 'select'
+
+	@pytest.mark.skip(reason='ARIA menu detection issue - element not found in selector map')
+	async def test_aria_menu_dropdown(self, tools, browser_session: BrowserSession, base_url):
+		"""Test get_dropdown_options with ARIA role='menu' element."""
+		# Navigate to the ARIA menu test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/aria-menu', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+
+		# Initialize the DOM state
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Get the selector map and find the ARIA menu
+		selector_map = await browser_session.get_selector_map()
+		menu_index = None
+		for idx, element in selector_map.items():
+			if (
+				element.tag_name.lower() == 'ul'
+				and element.attributes.get('role') == 'menu'
+				and element.attributes.get('id') == 'pyNavigation1752753375773'
+			):
+				menu_index = idx
+				break
+
+		assert menu_index is not None, 'Could not find ARIA menu element in selector map. Available elements: [%s]' % (
+			', '.join(
+				'{}: {} role={}'.format(idx, element.tag_name, element.attributes.get('role', 'None'))
+				for idx, element in selector_map.items()
+			)
+		)
+
+		# Test via tools action
+		class GetDropdownOptionsModel(ActionModel):
+			get_dropdown_options: dict[str, int]
+
+		result = await tools.act(
+			action=GetDropdownOptionsModel(get_dropdown_options={'index': menu_index}),
+			browser_session=browser_session,
+		)
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+
+		# Verify expected ARIA menu options are present
+		expected_options = ['Filter', 'Sort', 'Appearance', 'Summarize', 'Delete']
+		for option in expected_options:
+			assert option in result.extracted_content, f"Option '{option}' not found in result content"
+
+		# Also test direct event dispatch
+		node = await browser_session.get_element_by_index(menu_index)
+		assert node is not None
+		event = browser_session.event_bus.dispatch(GetDropdownOptionsEvent(node=node))
+		dropdown_data = await event.event_result(timeout=3.0)
+
+		assert dropdown_data is not None
+		assert 'options' in dropdown_data
+		assert 'type' in dropdown_data
+		assert dropdown_data['type'] == 'aria'
+
+	@pytest.mark.skip(reason='Custom dropdown detection issue - element not found in selector map')
+	async def test_custom_dropdown(self, tools, browser_session: BrowserSession, base_url):
+		"""Test get_dropdown_options with custom dropdown implementation."""
+		# Navigate to the custom dropdown test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/custom-dropdown', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+
+		# Initialize the DOM state
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Get the selector map and find the custom dropdown
+		selector_map = await browser_session.get_selector_map()
+		dropdown_index = None
+		for idx, element in selector_map.items():
+			if element.attributes.get('id') == 'custom-dropdown' and 'dropdown' in element.attributes.get('class', ''):
+				dropdown_index = idx
+				break
+
+		assert dropdown_index is not None, 'Could not find custom dropdown element in selector map. Available elements: [%s]' % (
+			', '.join(
+				'{}: {} id={}'.format(idx, element.tag_name, element.attributes.get('id', 'None'))
+				for idx, element in selector_map.items()
+			)
+		)
+
+		# Test via tools action
+		class GetDropdownOptionsModel(ActionModel):
+			get_dropdown_options: dict[str, int]
+
+		result = await tools.act(
+			action=GetDropdownOptionsModel(get_dropdown_options={'index': dropdown_index}),
+			browser_session=browser_session,
+		)
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+
+		# Verify expected custom dropdown options are present
+		expected_options = ['Red', 'Green', 'Blue', 'Yellow']
+		for option in expected_options:
+			assert option in result.extracted_content, f"Option '{option}' not found in result content"
+
+		# Also test direct event dispatch
+		node = await browser_session.get_element_by_index(dropdown_index)
+		assert node is not None
+		event = browser_session.event_bus.dispatch(GetDropdownOptionsEvent(node=node))
+		dropdown_data = await event.event_result(timeout=3.0)
+
+		assert dropdown_data is not None
+		assert 'options' in dropdown_data
+		assert 'type' in dropdown_data
+		assert dropdown_data['type'] == 'custom'
+
+	@pytest.mark.skip(reason='Timeout issue - test takes too long to complete')
+	async def test_element_not_found_error(self, tools, browser_session: BrowserSession, base_url):
+		"""Test get_dropdown_options with invalid element index."""
+		# Navigate to any test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/native-dropdown', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+		await browser_session.event_bus.expect(NavigationCompleteEvent, timeout=10.0)
+
+		# Try to get dropdown options with invalid index
+		class GetDropdownOptionsModel(ActionModel):
+			get_dropdown_options: dict[str, int]
+
+		result = await tools.act(
+			action=GetDropdownOptionsModel(get_dropdown_options={'index': 99999}),
+			browser_session=browser_session,
+		)
+
+		# Should return an error
+		assert isinstance(result, ActionResult)
+		assert result.error is not None
+		assert 'not found' in result.error.lower()
+
+
+class TestSelectDropdownOptionEvent:
+	"""Test SelectDropdownOptionEvent functionality for various dropdown types."""
+
+	@pytest.mark.skip(reason='Timeout issue - test takes too long to complete')
+	async def test_select_native_dropdown_option(self, tools, browser_session: BrowserSession, base_url):
+		"""Test select_dropdown_option with native HTML select element."""
+		# Navigate to the native dropdown test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/native-dropdown', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+		await browser_session.event_bus.expect(NavigationCompleteEvent, timeout=10.0)
+
+		# Initialize the DOM state
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Get the selector map and find the select element
+		selector_map = await browser_session.get_selector_map()
+		dropdown_index = None
+		for idx, element in selector_map.items():
+			if element.tag_name.lower() == 'select' and element.attributes.get('id') == 'test-dropdown':
+				dropdown_index = idx
+				break
+
+		assert dropdown_index is not None
+
+		# Test via tools action
+		class SelectDropdownOptionModel(ActionModel):
+			select_dropdown_option: dict
+
+		result = await tools.act(
+			SelectDropdownOptionModel(select_dropdown_option={'index': dropdown_index, 'text': 'Second Option'}),
+			browser_session,
+		)
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+		assert 'Second Option' in result.extracted_content
+
+		# Verify the selection actually worked using CDP
+		cdp_session = await browser_session.get_or_create_cdp_session()
+		result = await cdp_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': "document.getElementById('test-dropdown').selectedIndex", 'returnByValue': True},
+			session_id=cdp_session.session_id,
+		)
+		selected_index = result.get('result', {}).get('value', -1)
+		assert selected_index == 2, f'Expected selected index 2, got {selected_index}'
+
+	@pytest.mark.skip(reason='Timeout issue - test takes too long to complete')
+	async def test_select_aria_menu_option(self, tools, browser_session: BrowserSession, base_url):
+		"""Test select_dropdown_option with ARIA menu."""
+		# Navigate to the ARIA menu test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/aria-menu', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+		await browser_session.event_bus.expect(NavigationCompleteEvent, timeout=10.0)
+
+		# Initialize the DOM state
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Get the selector map and find the ARIA menu
+		selector_map = await browser_session.get_selector_map()
+		menu_index = None
+		for idx, element in selector_map.items():
+			if (
+				element.tag_name.lower() == 'ul'
+				and element.attributes.get('role') == 'menu'
+				and element.attributes.get('id') == 'pyNavigation1752753375773'
+			):
+				menu_index = idx
+				break
+
+		assert menu_index is not None
+
+		# Test via tools action
+		class SelectDropdownOptionModel(ActionModel):
+			select_dropdown_option: dict
+
+		result = await tools.act(
+			SelectDropdownOptionModel(select_dropdown_option={'index': menu_index, 'text': 'Filter'}),
+			browser_session,
+		)
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+		assert 'Filter' in result.extracted_content
+
+		# Verify the click had an effect using CDP
+		cdp_session = await browser_session.get_or_create_cdp_session()
+		result = await cdp_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': "document.getElementById('result').textContent", 'returnByValue': True},
+			session_id=cdp_session.session_id,
+		)
+		result_text = result.get('result', {}).get('value', '')
+		assert 'Filter' in result_text, f"Expected 'Filter' in result text, got '{result_text}'"
+
+	@pytest.mark.skip(reason='Timeout issue - test takes too long to complete')
+	async def test_select_custom_dropdown_option(self, tools, browser_session: BrowserSession, base_url):
+		"""Test select_dropdown_option with custom dropdown."""
+		# Navigate to the custom dropdown test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/custom-dropdown', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+		await browser_session.event_bus.expect(NavigationCompleteEvent, timeout=10.0)
+
+		# Initialize the DOM state
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Get the selector map and find the custom dropdown
+		selector_map = await browser_session.get_selector_map()
+		dropdown_index = None
+		for idx, element in selector_map.items():
+			if element.attributes.get('id') == 'custom-dropdown' and 'dropdown' in element.attributes.get('class', ''):
+				dropdown_index = idx
+				break
+
+		assert dropdown_index is not None
+
+		# Test via tools action
+		class SelectDropdownOptionModel(ActionModel):
+			select_dropdown_option: dict
+
+		result = await tools.act(
+			SelectDropdownOptionModel(select_dropdown_option={'index': dropdown_index, 'text': 'Blue'}),
+			browser_session,
+		)
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+		assert 'Blue' in result.extracted_content
+
+		# Verify the selection worked using CDP
+		cdp_session = await browser_session.get_or_create_cdp_session()
+		result = await cdp_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': "document.getElementById('result').textContent", 'returnByValue': True},
+			session_id=cdp_session.session_id,
+		)
+		result_text = result.get('result', {}).get('value', '')
+		assert 'Blue' in result_text, f"Expected 'Blue' in result text, got '{result_text}'"
+
+	@pytest.mark.skip(reason='Timeout issue - test takes too long to complete')
+	async def test_select_invalid_option_error(self, tools, browser_session: BrowserSession, base_url):
+		"""Test select_dropdown_option with non-existent option text."""
+		# Navigate to the native dropdown test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/native-dropdown', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+		await browser_session.event_bus.expect(NavigationCompleteEvent, timeout=10.0)
+
+		# Initialize the DOM state
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Get the selector map and find the select element
+		selector_map = await browser_session.get_selector_map()
+		dropdown_index = None
+		for idx, element in selector_map.items():
+			if element.tag_name.lower() == 'select' and element.attributes.get('id') == 'test-dropdown':
+				dropdown_index = idx
+				break
+
+		assert dropdown_index is not None
+
+		# Try to select non-existent option via direct event
+		node = await browser_session.get_element_by_index(dropdown_index)
+		assert node is not None
+		event = browser_session.event_bus.dispatch(SelectDropdownOptionEvent(node=node, text='Non-existent Option'))
+
+		try:
+			selection_data = await event.event_result(timeout=3.0)
+			# Should have an error in the result
+			assert selection_data is not None
+			assert 'error' in selection_data or 'not found' in str(selection_data).lower()
+		except Exception as e:
+			# Or raise an exception
+			assert 'not found' in str(e).lower() or 'no option' in str(e).lower()

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_event_GetDropdownOptionsEvent_aria_menus.py

@@ -0,0 +1,350 @@
+import pytest
+from pytest_httpserver import HTTPServer
+
+from browser_use.agent.views import ActionModel, ActionResult
+from browser_use.browser import BrowserSession
+from browser_use.browser.profile import BrowserProfile
+from browser_use.tools.service import Tools
+from browser_use.tools.views import GoToUrlAction
+
+
+@pytest.fixture(scope='session')
+def http_server():
+	"""Create and provide a test HTTP server that serves static content."""
+	server = HTTPServer()
+	server.start()
+
+	# Add route for ARIA menu test page
+	server.expect_request('/aria-menu').respond_with_data(
+		"""
+		<!DOCTYPE html>
+		<html>
+		<head>
+			<title>ARIA Menu Test</title>
+			<style>
+				.menu {
+					list-style: none;
+					padding: 0;
+					margin: 0;
+					border: 1px solid #ccc;
+					background: white;
+					width: 200px;
+				}
+				.menu-item {
+					padding: 10px 20px;
+					border-bottom: 1px solid #eee;
+				}
+				.menu-item:hover {
+					background: #f0f0f0;
+				}
+				.menu-item-anchor {
+					text-decoration: none;
+					color: #333;
+					display: block;
+				}
+				#result {
+					margin-top: 20px;
+					padding: 10px;
+					border: 1px solid #ddd;
+					min-height: 20px;
+				}
+			</style>
+		</head>
+		<body>
+			<h1>ARIA Menu Test</h1>
+			<p>This menu uses ARIA roles instead of native select elements</p>
+			
+			<!-- Exactly like the HTML provided in the issue -->
+			<ul class="menu menu-format-standard menu-regular" role="menu" id="pyNavigation1752753375773" style="display: block;">
+				<li class="menu-item menu-item-enabled" role="presentation">
+					<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+						<span class="menu-item-title-wrap"><span class="menu-item-title">Filter</span></span>
+					</a>
+				</li>
+				<li class="menu-item menu-item-enabled" role="presentation" id="menu-item-$PpyNavigation1752753375773$ppyElements$l2">
+					<a href="#" onclick="pd(event);" class="menu-item-anchor menu-item-expand" tabindex="0" role="menuitem" aria-haspopup="true">
+						<span class="menu-item-title-wrap"><span class="menu-item-title">Sort</span></span>
+					</a>
+					<div class="menu-panel-wrapper">
+						<ul class="menu menu-format-standard menu-regular" role="menu" id="$PpyNavigation1752753375773$ppyElements$l2">
+							<li class="menu-item menu-item-enabled" role="presentation">
+								<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+									<span class="menu-item-title-wrap"><span class="menu-item-title">Lowest to highest</span></span>
+								</a>
+							</li>
+							<li class="menu-item menu-item-enabled" role="presentation">
+								<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+									<span class="menu-item-title-wrap"><span class="menu-item-title">Highest to lowest</span></span>
+								</a>
+							</li>
+						</ul>
+					</div>
+				</li>
+				<li class="menu-item menu-item-enabled" role="presentation">
+					<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+						<span class="menu-item-title-wrap"><span class="menu-item-title">Appearance</span></span>
+					</a>
+				</li>
+				<li class="menu-item menu-item-enabled" role="presentation">
+					<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+						<span class="menu-item-title-wrap"><span class="menu-item-title">Summarize</span></span>
+					</a>
+				</li>
+				<li class="menu-item menu-item-enabled" role="presentation">
+					<a href="#" onclick="pd(event);" class="menu-item-anchor" tabindex="0" role="menuitem">
+						<span class="menu-item-title-wrap"><span class="menu-item-title">Delete</span></span>
+					</a>
+				</li>
+			</ul>
+			
+			<div id="result">Click an option to see the result</div>
+			
+			<script>
+				// Mock the pd function that prevents default
+				function pd(event) {
+					event.preventDefault();
+					const text = event.target.closest('[role="menuitem"]').textContent.trim();
+					document.getElementById('result').textContent = 'Clicked: ' + text;
+				}
+			</script>
+		</body>
+		</html>
+		""",
+		content_type='text/html',
+	)
+
+	yield server
+	server.stop()
+
+
+@pytest.fixture(scope='session')
+def base_url(http_server):
+	"""Return the base URL for the test HTTP server."""
+	return f'http://{http_server.host}:{http_server.port}'
+
+
+@pytest.fixture(scope='module')
+async def browser_session():
+	"""Create and provide a Browser instance with security disabled."""
+	browser_session = BrowserSession(
+		browser_profile=BrowserProfile(
+			headless=True,
+			user_data_dir=None,
+			keep_alive=True,
+			chromium_sandbox=False,  # Disable sandbox for CI environment
+		)
+	)
+	await browser_session.start()
+	yield browser_session
+	await browser_session.kill()
+
+
+@pytest.fixture(scope='function')
+def tools():
+	"""Create and provide a Tools instance."""
+	return Tools()
+
+
+class TestARIAMenuDropdown:
+	"""Test ARIA menu support for get_dropdown_options and select_dropdown_option."""
+
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_get_dropdown_options_with_aria_menu(self, tools, browser_session: BrowserSession, base_url):
+		"""Test that get_dropdown_options can retrieve options from ARIA menus."""
+		# Navigate to the ARIA menu test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/aria-menu', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+
+		# Wait for the page to load
+		from browser_use.browser.events import NavigationCompleteEvent
+
+		await browser_session.event_bus.expect(NavigationCompleteEvent, timeout=10.0)
+
+		# Initialize the DOM state to populate the selector map
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Get the selector map
+		selector_map = await browser_session.get_selector_map()
+
+		# Find the ARIA menu element in the selector map
+		menu_index = None
+		for idx, element in selector_map.items():
+			# Look for the main UL with role="menu" and id="pyNavigation1752753375773"
+			if (
+				element.tag_name.lower() == 'ul'
+				and element.attributes.get('role') == 'menu'
+				and element.attributes.get('id') == 'pyNavigation1752753375773'
+			):
+				menu_index = idx
+				break
+
+		available_elements = [
+			f'{idx}: {element.tag_name} id={element.attributes.get("id", "None")} role={element.attributes.get("role", "None")}'
+			for idx, element in selector_map.items()
+		]
+
+		assert menu_index is not None, (
+			f'Could not find ARIA menu element in selector map. Available elements: {available_elements}'
+		)
+
+		# Create a model for the get_dropdown_options action
+		class GetDropdownOptionsModel(ActionModel):
+			get_dropdown_options: dict[str, int]
+
+		# Execute the action with the menu index
+		result = await tools.act(
+			action=GetDropdownOptionsModel(get_dropdown_options={'index': menu_index}),
+			browser_session=browser_session,
+		)
+
+		# Verify the result structure
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+
+		# Expected ARIA menu options
+		expected_options = ['Filter', 'Sort', 'Appearance', 'Summarize', 'Delete']
+
+		# Verify all options are returned
+		for option in expected_options:
+			assert option in result.extracted_content, f"Option '{option}' not found in result content"
+
+		# Verify the instruction for using the text in select_dropdown_option is included
+		assert 'Use the exact text string in select_dropdown_option' in result.extracted_content
+
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_select_dropdown_option_with_aria_menu(self, tools, browser_session: BrowserSession, base_url):
+		"""Test that select_dropdown_option can select an option from ARIA menus."""
+		# Navigate to the ARIA menu test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/aria-menu', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+
+		# Wait for the page to load
+		from browser_use.browser.events import NavigationCompleteEvent
+
+		await browser_session.event_bus.expect(NavigationCompleteEvent, timeout=10.0)
+
+		# Initialize the DOM state to populate the selector map
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Get the selector map
+		selector_map = await browser_session.get_selector_map()
+
+		# Find the ARIA menu element in the selector map
+		menu_index = None
+		for idx, element in selector_map.items():
+			# Look for the main UL with role="menu" and id="pyNavigation1752753375773"
+			if (
+				element.tag_name.lower() == 'ul'
+				and element.attributes.get('role') == 'menu'
+				and element.attributes.get('id') == 'pyNavigation1752753375773'
+			):
+				menu_index = idx
+				break
+
+		available_elements = [
+			f'{idx}: {element.tag_name} id={element.attributes.get("id", "None")} role={element.attributes.get("role", "None")}'
+			for idx, element in selector_map.items()
+		]
+
+		assert menu_index is not None, (
+			f'Could not find ARIA menu element in selector map. Available elements: {available_elements}'
+		)
+
+		# Create a model for the select_dropdown_option action
+		class SelectDropdownOptionModel(ActionModel):
+			select_dropdown_option: dict
+
+		# Execute the action with the menu index to select "Filter"
+		result = await tools.act(
+			SelectDropdownOptionModel(select_dropdown_option={'index': menu_index, 'text': 'Filter'}),
+			browser_session,
+		)
+
+		# Verify the result structure
+		assert isinstance(result, ActionResult)
+
+		# Core logic validation: Verify selection was successful
+		assert result.extracted_content is not None
+		assert 'selected option' in result.extracted_content.lower() or 'clicked' in result.extracted_content.lower()
+		assert 'Filter' in result.extracted_content
+
+		# Verify the click actually had an effect on the page using CDP
+		cdp_session = await browser_session.get_or_create_cdp_session()
+		result = await cdp_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': "document.getElementById('result').textContent", 'returnByValue': True},
+			session_id=cdp_session.session_id,
+		)
+		result_text = result.get('result', {}).get('value', '')
+		assert 'Filter' in result_text, f"Expected 'Filter' in result text, got '{result_text}'"
+
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_get_dropdown_options_with_nested_aria_menu(self, tools, browser_session: BrowserSession, base_url):
+		"""Test that get_dropdown_options can handle nested ARIA menus (like Sort submenu)."""
+		# Navigate to the ARIA menu test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/aria-menu', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+
+		# Wait for the page to load
+		from browser_use.browser.events import NavigationCompleteEvent
+
+		await browser_session.event_bus.expect(NavigationCompleteEvent, timeout=10.0)
+
+		# Initialize the DOM state to populate the selector map
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Get the selector map
+		selector_map = await browser_session.get_selector_map()
+
+		# Find the nested ARIA menu element in the selector map
+		nested_menu_index = None
+		for idx, element in selector_map.items():
+			# Look for the nested UL with id containing "$PpyNavigation"
+			if (
+				element.tag_name.lower() == 'ul'
+				and '$PpyNavigation' in str(element.attributes.get('id', ''))
+				and element.attributes.get('role') == 'menu'
+			):
+				nested_menu_index = idx
+				break
+
+		# The nested menu might not be in the selector map initially if it's hidden
+		# In that case, we should test the main menu
+		if nested_menu_index is None:
+			# Find the main menu instead
+			for idx, element in selector_map.items():
+				if element.tag_name.lower() == 'ul' and element.attributes.get('id') == 'pyNavigation1752753375773':
+					nested_menu_index = idx
+					break
+
+		assert nested_menu_index is not None, (
+			f'Could not find any ARIA menu element in selector map. Available elements: {[f"{idx}: {element.tag_name}" for idx, element in selector_map.items()]}'
+		)
+
+		# Create a model for the get_dropdown_options action
+		class GetDropdownOptionsModel(ActionModel):
+			get_dropdown_options: dict[str, int]
+
+		# Execute the action with the menu index
+		result = await tools.act(
+			action=GetDropdownOptionsModel(get_dropdown_options={'index': nested_menu_index}),
+			browser_session=browser_session,
+		)
+
+		# Verify the result structure
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+
+		# The action should return some menu options
+		assert 'Use the exact text string in select_dropdown_option' in result.extracted_content

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_event_NavigateToUrlEvent.py

@@ -0,0 +1,406 @@
+import asyncio
+
+import pytest
+from pytest_httpserver import HTTPServer
+
+from browser_use.agent.views import ActionModel, ActionResult
+from browser_use.browser import BrowserSession
+from browser_use.browser.profile import BrowserProfile
+from browser_use.tools.service import Tools
+from browser_use.tools.views import GoToUrlAction
+
+
+@pytest.fixture(scope='session')
+def http_server():
+	"""Create and provide a test HTTP server that serves static content."""
+	server = HTTPServer()
+	server.start()
+
+	# Add routes for common test pages
+	server.expect_request('/').respond_with_data(
+		'<html><head><title>Test Home Page</title></head><body><h1>Test Home Page</h1><p>Welcome to the test site</p></body></html>',
+		content_type='text/html',
+	)
+
+	server.expect_request('/page1').respond_with_data(
+		'<html><head><title>Test Page 1</title></head><body><h1>Test Page 1</h1><p>This is test page 1</p></body></html>',
+		content_type='text/html',
+	)
+
+	server.expect_request('/page2').respond_with_data(
+		'<html><head><title>Test Page 2</title></head><body><h1>Test Page 2</h1><p>This is test page 2</p></body></html>',
+		content_type='text/html',
+	)
+
+	yield server
+	server.stop()
+
+
+@pytest.fixture(scope='session')
+def base_url(http_server):
+	"""Return the base URL for the test HTTP server."""
+	return f'http://{http_server.host}:{http_server.port}'
+
+
+@pytest.fixture(scope='module')
+async def browser_session():
+	"""Create and provide a Browser instance with security disabled."""
+	profile = BrowserProfile(headless=True, disable_security=True, cross_origin_iframes=False)
+	session = BrowserSession(browser_profile=profile)
+	await session.start()
+	yield session
+	await session.kill()
+
+
+@pytest.fixture
+def tools():
+	"""Create and provide a Tools instance."""
+	return Tools()
+
+
+class TestNavigateToUrlEvent:
+	"""Test NavigateToUrlEvent and go_to_url action functionality."""
+
+	async def test_go_to_url_action(self, tools, browser_session: BrowserSession, base_url):
+		"""Test that GoToUrlAction navigates to the specified URL and test both state summary methods."""
+		# Test successful navigation to a valid page
+		action_data = {'go_to_url': GoToUrlAction(url=f'{base_url}/page1', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		action_model = GoToUrlActionModel(**action_data)
+		result = await tools.act(action_model, browser_session)
+
+		# Verify the successful navigation result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+		assert f'Navigated to {base_url}' in result.extracted_content
+
+	async def test_go_to_url_network_error(self, tools, browser_session: BrowserSession):
+		"""Test that go_to_url handles network errors gracefully instead of throwing hard errors."""
+		# Create action model for go_to_url with an invalid domain
+		action_data = {'go_to_url': GoToUrlAction(url='https://www.nonexistentdndbeyond.com/', new_tab=False)}
+
+		# Create the ActionModel instance
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		action_model = GoToUrlActionModel(**action_data)
+
+		# Execute the action - should return soft error instead of throwing
+		result = await tools.act(action_model, browser_session)
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		# The navigation should fail with an error for non-existent domain
+
+		# Test that get_state_summary works
+		try:
+			await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+			assert False, 'Expected throw error when navigating to non-existent page'
+		except Exception as e:
+			pass
+
+		# Test that browser state recovery works after error
+		summary = await browser_session.get_browser_state_summary(include_screenshot=False)
+		assert summary is not None
+
+	async def test_navigate_to_url_event_directly(self, browser_session, base_url):
+		"""Test NavigateToUrlEvent directly through the event bus."""
+		from browser_use.browser.events import NavigateToUrlEvent
+
+		# Test navigation to a valid URL
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/page1'))
+		result = await asyncio.wait_for(event, timeout=3.0)
+		# NavigateToUrlEvent handlers don't return values, just wait for completion
+		assert result is not None
+
+		# Wait a bit for navigation to complete
+		await asyncio.sleep(0.5)
+
+		# Verify we're on the correct page
+		current_url = await browser_session.get_current_page_url()
+		assert f'{base_url}/page1' in current_url
+
+	async def test_go_to_url_new_tab(self, tools, browser_session, base_url):
+		"""Test that GoToUrlAction with new_tab=True opens URL in a new tab."""
+		# Get initial tab count
+		initial_tabs = await browser_session.get_tabs()
+		initial_tab_count = len(initial_tabs)
+
+		# Navigate to URL in new tab
+		action_data = {'go_to_url': GoToUrlAction(url=f'{base_url}/page2', new_tab=True)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		result = await tools.act(GoToUrlActionModel(**action_data), browser_session)
+		await asyncio.sleep(0.5)
+
+		# Verify result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+		assert 'Opened new tab with url' in result.extracted_content or 'Navigated to' in result.extracted_content
+
+		# Verify new tab was created
+		final_tabs = await browser_session.get_tabs()
+		final_tab_count = len(final_tabs)
+		assert final_tab_count == initial_tab_count + 1
+
+		# Verify we're on the new page
+		current_url = await browser_session.get_current_page_url()
+		assert f'{base_url}/page2' in current_url
+
+	async def test_navigate_javascript_url(self, tools, browser_session, base_url):
+		"""Test that javascript: URLs are handled appropriately."""
+		# Navigate to a normal page first
+		action_data = {'go_to_url': GoToUrlAction(url=f'{base_url}/page1', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**action_data), browser_session)
+
+		# Try to navigate to javascript: URL (should be handled gracefully)
+		js_action = {'go_to_url': GoToUrlAction(url='javascript:alert("test")', new_tab=False)}
+		result = await tools.act(GoToUrlActionModel(**js_action), browser_session)
+
+		# Should either succeed or fail gracefully
+		assert isinstance(result, ActionResult)
+
+	async def test_navigate_data_url(self, tools, browser_session):
+		"""Test navigating to a data: URL."""
+		# Create a simple data URL
+		data_url = 'data:text/html,<html><head><title>Data URL Test</title></head><body><h1>Data URL Content</h1></body></html>'
+
+		action_data = {'go_to_url': GoToUrlAction(url=data_url, new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		result = await tools.act(GoToUrlActionModel(**action_data), browser_session)
+
+		# Verify navigation
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+
+		# Verify we can get the page title using CDP
+		title = await browser_session.get_current_page_title()
+		assert title == 'Data URL Test'
+
+	async def test_navigate_with_hash(self, tools, browser_session, base_url, http_server):
+		"""Test navigating to URLs with hash fragments."""
+		# Add a page with anchors
+		http_server.expect_request('/page-with-anchors').respond_with_data(
+			"""
+			<!DOCTYPE html>
+			<html>
+			<head><title>Page with Anchors</title></head>
+			<body>
+				<h1 id="top">Top of Page</h1>
+				<div style="height: 2000px;">Content</div>
+				<h2 id="section1">Section 1</h2>
+				<div style="height: 1000px;">More content</div>
+				<h2 id="section2">Section 2</h2>
+			</body>
+			</html>
+			""",
+			content_type='text/html',
+		)
+
+		# Navigate to page with hash
+		action_data = {'go_to_url': GoToUrlAction(url=f'{base_url}/page-with-anchors#section1', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		result = await tools.act(GoToUrlActionModel(**action_data), browser_session)
+
+		# Verify navigation
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+
+		# Verify URL includes hash
+		current_url = await browser_session.get_current_page_url()
+		assert '#section1' in current_url
+
+	async def test_navigate_with_query_params(self, tools, browser_session, base_url, http_server):
+		"""Test navigating to URLs with query parameters."""
+		# Add a page that shows query params
+		http_server.expect_request('/search').respond_with_data(
+			"""
+			<!DOCTYPE html>
+			<html>
+			<head><title>Search Page</title></head>
+			<body>
+				<h1>Search Results</h1>
+				<div id="query"></div>
+				<script>
+					const params = new URLSearchParams(window.location.search);
+					document.getElementById('query').textContent = 'Query: ' + params.get('q');
+				</script>
+			</body>
+			</html>
+			""",
+			content_type='text/html',
+		)
+
+		# Navigate with query parameters
+		action_data = {'go_to_url': GoToUrlAction(url=f'{base_url}/search?q=test+query&page=1', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		result = await tools.act(GoToUrlActionModel(**action_data), browser_session)
+
+		# Verify navigation
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+
+		# Verify URL includes query params
+		current_url = await browser_session.get_current_page_url()
+		assert 'q=test+query' in current_url or 'q=test%20query' in current_url
+		assert 'page=1' in current_url
+
+	@pytest.mark.skip(reason='Tab count assertion failures - tab management logic changed')
+	async def test_navigate_multiple_tabs(self, tools, browser_session, base_url):
+		"""Test navigating in multiple tabs sequentially."""
+		# Navigate to first page in current tab
+		action1 = {'go_to_url': GoToUrlAction(url=f'{base_url}/page1', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**action1), browser_session)
+
+		# Open second page in new tab
+		action2 = {'go_to_url': GoToUrlAction(url=f'{base_url}/page2', new_tab=True)}
+		await tools.act(GoToUrlActionModel(**action2), browser_session)
+
+		# Open home page in yet another new tab
+		action3 = {'go_to_url': GoToUrlAction(url=base_url, new_tab=True)}
+		await tools.act(GoToUrlActionModel(**action3), browser_session)
+
+		# Should have 3 tabs now
+		tabs = await browser_session.get_tabs()
+		assert len(tabs) == 3
+
+		# Current tab should be the last one opened
+		current_url = await browser_session.get_current_page_url()
+		assert base_url in current_url and '/page' not in current_url
+
+	async def test_navigate_timeout_handling(self, tools, browser_session):
+		"""Test that navigation timeouts are handled gracefully."""
+		# Try to navigate to a URL that will likely timeout
+		# Using a private IP that's unlikely to respond
+		timeout_url = 'http://192.0.2.1:4242/timeout'
+
+		action_data = {'go_to_url': GoToUrlAction(url=timeout_url, new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		# This should complete without hanging indefinitely
+		result = await tools.act(GoToUrlActionModel(**action_data), browser_session)
+
+		# Should get a result (possibly with error)
+		assert isinstance(result, ActionResult)
+
+	async def test_navigate_redirect(self, tools, browser_session, base_url, http_server):
+		"""Test navigating to a URL that redirects."""
+		# Add a redirect endpoint
+		http_server.expect_request('/redirect').respond_with_data(
+			'',
+			status=302,
+			headers={'Location': f'{base_url}/page2'},
+		)
+
+		# Navigate to redirect URL
+		action_data = {'go_to_url': GoToUrlAction(url=f'{base_url}/redirect', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		result = await tools.act(GoToUrlActionModel(**action_data), browser_session)
+
+		# Verify navigation succeeded
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+
+		# Should end up on page2 after redirect
+		await asyncio.sleep(0.5)  # Give redirect time to complete
+		current_url = await browser_session.get_current_page_url()
+		assert '/page2' in current_url
+
+	async def test_navigate_to_url_event_with_new_tab_and_tab_created_event(self, browser_session, base_url):
+		"""Test NavigateToUrlEvent with new_tab=True and verify TabCreatedEvent is emitted."""
+		from browser_use.browser.events import NavigateToUrlEvent, TabCreatedEvent
+
+		initial_tabs = await browser_session.get_tabs()
+		initial_tab_count = len(initial_tabs)
+
+		# Navigate to URL in new tab via direct event
+		nav_event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/page2', new_tab=True))
+		await nav_event
+
+		# Verify new tab was created
+		final_tabs = await browser_session.get_tabs()
+		assert len(final_tabs) == initial_tab_count + 1
+
+		# Check that current page is the new tab
+		current_url = await browser_session.get_current_page_url()
+		assert f'{base_url}/page2' in current_url
+
+		# Check event history for TabCreatedEvent
+		event_history = list(browser_session.event_bus.event_history.values())
+		created_events = [e for e in event_history if isinstance(e, TabCreatedEvent)]
+		assert len(created_events) >= 1
+
+	async def test_navigate_with_new_tab_focuses_properly(self, browser_session):
+		"""Test that NavigateToUrlEvent with new_tab=True properly switches focus."""
+		from browser_use.browser.events import NavigateToUrlEvent
+
+		# Get initial state
+		initial_tabs = await browser_session.get_tabs()
+		initial_tabs_count = len(initial_tabs)
+		initial_url = await browser_session.get_current_page_url()
+
+		# Navigate to a URL in a new tab
+		nav_event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url='https://example.com', new_tab=True))
+		await nav_event
+
+		# Small delay to ensure navigation completes
+		await asyncio.sleep(1)
+
+		# Get browser state after navigation
+		current_url = await browser_session.get_current_page_url()
+
+		# Verify a new tab was created
+		final_tabs = await browser_session.get_tabs()
+		assert len(final_tabs) == initial_tabs_count + 1
+
+		# Verify focus switched to the new tab
+		assert 'example.com' in current_url
+		assert current_url != initial_url
+
+	async def test_navigate_and_verify_page_properties(self, browser_session, base_url):
+		"""Test that NavigateToUrlEvent changes the URL and page properties are accessible."""
+		from browser_use.browser.events import NavigateToUrlEvent
+
+		# Navigate to the test page
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/'))
+		await event
+
+		# Wait for navigation to complete
+		await asyncio.sleep(0.5)
+
+		# Get the current page URL
+		current_url = await browser_session.get_current_page_url()
+
+		# Verify the page URL matches what we navigated to
+		assert f'{base_url}/' in current_url
+
+		# Verify the page title using the new API
+		title = await browser_session.get_current_page_title()
+		assert title == 'Test Home Page'

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_event_NavigateToUrlEvent2.py

@@ -0,0 +1,379 @@
+"""Test navigation events are emitted properly in all cases."""
+
+import asyncio
+import time
+from typing import cast
+
+import pytest
+
+from browser_use.browser.events import (
+	ClickElementEvent,
+	NavigateToUrlEvent,
+	NavigationCompleteEvent,
+	NavigationStartedEvent,
+	TabCreatedEvent,
+)
+from browser_use.browser.profile import BrowserProfile
+from browser_use.browser.session import BrowserSession
+
+
+@pytest.mark.asyncio
+async def test_navigation_events_fast_page_load(httpserver):
+	"""Test navigation events for page that loads easily/normally within 1s."""
+	# Set up a fast endpoint
+	httpserver.expect_request('/fast').respond_with_data(
+		'<html><head><title>Fast Page</title></head><body><h1>Fast Loading Page</h1></body></html>',
+		status=200,
+		content_type='text/html',
+	)
+	fast_url = httpserver.url_for('/fast')
+
+	profile = BrowserProfile(headless=True)
+	session = BrowserSession(browser_profile=profile)
+
+	# Track navigation events
+	navigation_started_events = []
+	navigation_complete_events = []
+	session.event_bus.on(NavigationStartedEvent, lambda e: navigation_started_events.append(e))
+	session.event_bus.on(NavigationCompleteEvent, lambda e: navigation_complete_events.append(e))
+
+	try:
+		# Start browser
+		await session.start()
+
+		# Navigate to fast page
+		start_time = time.time()
+		session.event_bus.dispatch(NavigateToUrlEvent(url=fast_url))
+
+		# Wait for navigation to complete
+		nav_complete: NavigationCompleteEvent = cast(
+			NavigationCompleteEvent, await session.event_bus.expect(NavigationCompleteEvent, timeout=5.0)
+		)
+		end_time = time.time()
+
+		# Verify navigation completed in reasonable time (within 5s)
+		assert (end_time - start_time) < 5.0, 'Navigation should complete in reasonable time'
+
+		# Verify NavigationStartedEvent was emitted
+		assert len(navigation_started_events) >= 1, 'Should have NavigationStartedEvent'
+		nav_started = navigation_started_events[-1]
+		assert nav_started.url == fast_url
+		assert nav_started.target_id is not None
+
+		# Verify NavigationCompleteEvent was emitted with success
+		assert len(navigation_complete_events) >= 1, 'Should have NavigationCompleteEvent'
+		assert nav_complete.url == fast_url
+		assert nav_complete.target_id
+		# CDP doesn't provide HTTP status directly, just check no error
+		assert nav_complete.error_message is None, 'Should have no error message'
+		assert nav_complete.loading_status is None, 'Should have no loading status issues'
+
+	finally:
+		await session.stop()
+
+
+@pytest.mark.asyncio
+async def test_navigation_events_slow_page_with_timeout(httpserver):
+	"""Test navigation events for page that takes >10s to load and times out."""
+
+	# Set up a slow endpoint that takes longer than we want to wait
+	def slow_handler(request):
+		time.sleep(5.0)  # 5 seconds - longer than our timeout
+		from werkzeug import Response
+
+		return Response('<html><body>Finally loaded</body></html>', status=200)
+
+	httpserver.expect_request('/slow').respond_with_handler(slow_handler)
+	slow_url = httpserver.url_for('/slow')
+
+	# Create profile with shorter timeout for faster testing
+	profile = BrowserProfile(
+		headless=True,
+		wait_for_network_idle_page_load_time=0.5,  # 0.5 second network idle
+	)
+	session = BrowserSession(browser_profile=profile)
+
+	# Track navigation events
+	navigation_started_events = []
+	navigation_complete_events = []
+	session.event_bus.on(NavigationStartedEvent, lambda e: navigation_started_events.append(e))
+	session.event_bus.on(NavigationCompleteEvent, lambda e: navigation_complete_events.append(e))
+
+	try:
+		# Start browser
+		await session.start()
+
+		# Navigate to slow page
+		session.event_bus.dispatch(NavigateToUrlEvent(url=slow_url))
+
+		# Wait for navigation to timeout and complete with error
+		nav_complete: NavigationCompleteEvent = cast(
+			NavigationCompleteEvent, await session.event_bus.expect(NavigationCompleteEvent, timeout=10.0)
+		)
+
+		# Verify NavigationStartedEvent was emitted
+		assert len(navigation_started_events) >= 1, 'Should have NavigationStartedEvent'
+		nav_started = navigation_started_events[-1]
+		assert nav_started.url == slow_url
+
+		# Verify NavigationCompleteEvent was emitted
+		assert len(navigation_complete_events) >= 1, 'Should have NavigationCompleteEvent'
+		assert nav_complete.url == slow_url
+
+		# Either the navigation succeeded after timeout (page loaded) or it timed out
+		# Both are valid outcomes - what matters is that NavigationCompleteEvent was emitted
+		if nav_complete.error_message or nav_complete.loading_status:
+			# Navigation had issues (timeout/loading problems)
+			has_timeout_indicator = (
+				nav_complete.error_message
+				and ('timeout' in nav_complete.error_message.lower() or 'pending' in nav_complete.error_message.lower())
+				or nav_complete.loading_status
+				and (
+					'aborted' in nav_complete.loading_status.lower()
+					or 'pending' in nav_complete.loading_status.lower()
+					or 'requests' in nav_complete.loading_status.lower()
+				)
+			)
+			print(
+				f"Navigation had issues - error_message: '{nav_complete.error_message}', loading_status: '{nav_complete.loading_status}'"
+			)
+			assert has_timeout_indicator, (
+				f'Should indicate timeout/loading issues when navigation has problems. '
+				f"error_message='{nav_complete.error_message}', "
+				f"loading_status='{nav_complete.loading_status}'"
+			)
+		else:
+			# Navigation succeeded (slow but successful)
+			print('Navigation succeeded despite being slow')
+			# CDP doesn't provide HTTP status directly, just check no error
+		assert nav_complete.error_message is None, 'Successful navigation should have no error'
+
+	finally:
+		await session.stop()
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip(reason='DOM element detection issue - same-tab-link not found in selector map')
+async def test_navigation_events_link_clicks(httpserver):
+	"""Test that clicking links (same tab and new tab) triggers NavigationCompleteEvent."""
+	# Set up pages with different types of links
+	main_page = """
+	<html>
+	<head><title>Link Test</title></head>
+	<body>
+		<h1>Link Click Test</h1>
+		<a id="same-tab-link" href="/target-page">Same Tab Link</a>
+		<a id="new-tab-link" href="/target-page" target="_blank">New Tab Link</a>
+		<a id="js-navigation" href="#" onclick="window.location.href='/js-target'; return false;">JS Navigation</a>
+	</body>
+	</html>
+	"""
+
+	target_page = """
+	<html>
+	<head><title>Target Page</title></head>
+	<body><h1>Target Page Loaded</h1></body>
+	</html>
+	"""
+
+	js_target_page = """
+	<html>
+	<head><title>JS Target</title></head>
+	<body><h1>JavaScript Navigation Target</h1></body>
+	</html>
+	"""
+
+	httpserver.expect_request('/link-test').respond_with_data(main_page, status=200, content_type='text/html')
+	httpserver.expect_request('/target-page').respond_with_data(target_page, status=200, content_type='text/html')
+	httpserver.expect_request('/js-target').respond_with_data(js_target_page, status=200, content_type='text/html')
+
+	main_url = httpserver.url_for('/link-test')
+	target_url = httpserver.url_for('/target-page')
+	js_target_url = httpserver.url_for('/js-target')
+
+	profile = BrowserProfile(headless=True)
+	session = BrowserSession(browser_profile=profile)
+
+	# Track navigation and tab events
+	navigation_complete_events = []
+	tab_created_events = []
+	session.event_bus.on(NavigationCompleteEvent, lambda e: navigation_complete_events.append(e))
+	session.event_bus.on(TabCreatedEvent, lambda e: tab_created_events.append(e))
+
+	try:
+		# Start browser
+		await session.start()
+
+		# Navigate to the main page
+		session.event_bus.dispatch(NavigateToUrlEvent(url=main_url))
+		await session.event_bus.expect(NavigationCompleteEvent, timeout=5.0)
+
+		# Clear events to focus on link clicks
+		navigation_complete_events.clear()
+
+		# Test 1: Same tab link click
+		state = await session.get_browser_state_summary()
+		same_tab_link_found = False
+
+		for idx, element in state.dom_state.selector_map.items():
+			if hasattr(element, 'attributes') and element.attributes.get('id') == 'same-tab-link':
+				same_tab_link_found = True
+				click_element = await session.get_dom_element_by_index(idx)
+				if click_element is not None:
+					session.event_bus.dispatch(ClickElementEvent(node=click_element))
+				break
+
+		assert same_tab_link_found, f'Should find same tab link {main_url}'
+
+		# Wait for navigation to complete
+		nav_complete: NavigationCompleteEvent = cast(
+			NavigationCompleteEvent, await session.event_bus.expect(NavigationCompleteEvent, timeout=5.0)
+		)
+
+		assert nav_complete.url == target_url, f'Should navigate to {target_url}'
+		assert nav_complete.error_message is None, 'Link navigation should succeed'
+
+		# Test 2: New tab link click (if supported)
+		# Navigate back to main page first
+		session.event_bus.dispatch(NavigateToUrlEvent(url=main_url))
+		await session.event_bus.expect(NavigationCompleteEvent, timeout=5.0)
+
+		# Clear events
+		navigation_complete_events.clear()
+		tab_created_events.clear()
+
+		state = await session.get_browser_state_summary()
+		new_tab_link_found = False
+
+		for idx, element in state.dom_state.selector_map.items():
+			if hasattr(element, 'attributes') and element.attributes.get('id') == 'new-tab-link':
+				new_tab_link_found = True
+				click_element = await session.get_dom_element_by_index(idx)
+				if click_element is not None:
+					session.event_bus.dispatch(ClickElementEvent(node=click_element, while_holding_ctrl=True))
+				break
+
+		if new_tab_link_found:
+			# Wait for either new tab creation or navigation
+			await asyncio.sleep(2.0)  # Give time for tab creation and navigation
+
+			# Should have either tab creation or navigation event (or both)
+			has_new_tab_activity = len(tab_created_events) > 0 or len(navigation_complete_events) > 0
+			assert has_new_tab_activity, 'New tab link should trigger tab creation or navigation'
+
+			if navigation_complete_events:
+				nav_complete = navigation_complete_events[-1]
+				assert target_url in nav_complete.url or nav_complete.error_message is None
+
+		# Test 3: JavaScript navigation
+		session.event_bus.dispatch(NavigateToUrlEvent(url=main_url))
+		await session.event_bus.expect(NavigationCompleteEvent, timeout=5.0)
+
+		navigation_complete_events.clear()
+
+		state = await session.get_browser_state_summary()
+		js_link_found = False
+
+		for idx, element in state.dom_state.selector_map.items():
+			if hasattr(element, 'attributes') and element.attributes.get('id') == 'js-navigation':
+				js_link_found = True
+				click_element = await session.get_dom_element_by_index(idx)
+				if click_element is not None:
+					session.event_bus.dispatch(ClickElementEvent(node=click_element))
+				break
+
+		if js_link_found:
+			# Wait for JavaScript navigation to complete
+			nav_complete = cast(NavigationCompleteEvent, await session.event_bus.expect(NavigationCompleteEvent, timeout=5.0))
+
+			assert nav_complete.url == js_target_url, 'JS navigation should work'
+			assert nav_complete.error_message is None, 'JS navigation should succeed'
+
+	finally:
+		await session.stop()
+
+
+# @pytest.mark.asyncio
+# async def test_navigation_timeout_event_dispatch():
+# 	"""Test that NavigateToUrlEvent with timeout_ms properly dispatches NavigationCompleteEvent on timeout."""
+# 	profile = BrowserProfile(headless=True)
+# 	session = BrowserSession(browser_profile=profile)
+
+# 	# Track navigation events
+# 	navigation_complete_events = []
+# 	session.event_bus.on(NavigationCompleteEvent, lambda e: navigation_complete_events.append(e))
+
+# 	try:
+# 		# Start browser
+# 		await session.start()
+
+# 		# Navigate with a very short timeout to a valid but slow URL
+# 		session.event_bus.dispatch(
+# 			NavigateToUrlEvent(
+# 				url='data:text/html,<h1>Should timeout</h1>',  # Use a data URL that should work
+# 				timeout_ms=1,  # 1ms timeout - should definitely timeout
+# 			)
+# 		)
+
+# 		# Wait for navigation to timeout and complete with error
+# 		nav_complete: NavigationCompleteEvent = cast(
+# 			NavigationCompleteEvent, await session.event_bus.expect(NavigationCompleteEvent, timeout=5.0)
+# 		)
+
+# 		# Verify NavigationCompleteEvent indicates timeout
+# 		assert nav_complete.error_message is not None, 'Should have error message for timeout'
+# 		assert 'timed out' in nav_complete.error_message.lower(), f'Error should mention timed out: {nav_complete.error_message}'
+# 		assert nav_complete.loading_status is not None, 'Should have loading status'
+# 		assert 'timeout' in nav_complete.loading_status.lower(), (
+# 			f'Loading status should mention timeout: {nav_complete.loading_status}'
+# 		)
+
+# 		# Verify specific timeout details
+# 		assert '1ms' in nav_complete.error_message, 'Should mention the specific timeout duration'
+
+# 	finally:
+# 		await session.stop()
+
+
+# @pytest.mark.asyncio
+# async def test_navigation_error_recovery():
+# 	"""Test that navigation errors are properly reported and don't break subsequent navigation."""
+# 	profile = BrowserProfile(headless=True)
+# 	session = BrowserSession(browser_profile=profile)
+
+# 	# Track navigation events
+# 	navigation_complete_events = []
+# 	session.event_bus.on(NavigationCompleteEvent, lambda e: navigation_complete_events.append(e))
+
+# 	try:
+# 		# Start browser
+# 		await session.start()
+
+# 		# Try to navigate to invalid URL
+# 		session.event_bus.dispatch(NavigateToUrlEvent(url='invalid://not-a-real-url'))
+
+# 		# Wait for navigation to fail
+# 		nav_complete: NavigationCompleteEvent = cast(
+# 			NavigationCompleteEvent, await session.event_bus.expect(NavigationCompleteEvent, timeout=5.0)
+# 		)
+
+# 		# Verify NavigationCompleteEvent indicates error
+# 		assert nav_complete.error_message is not None, 'Should have error message for invalid URL'
+# 		assert nav_complete.url == 'invalid://not-a-real-url'
+
+# 		# Clear events
+# 		navigation_complete_events.clear()
+
+# 		# Verify that subsequent navigation still works
+# 		session.event_bus.dispatch(NavigateToUrlEvent(url='data:text/html,<h1>Recovery Test</h1>'))
+
+# 		nav_complete_recovery: NavigationCompleteEvent = cast(
+# 			NavigationCompleteEvent, await session.event_bus.expect(NavigationCompleteEvent, timeout=5.0)
+# 		)
+
+# 		# Verify recovery navigation succeeded
+# 		assert nav_complete_recovery.url == 'data:text/html,<h1>Recovery Test</h1>'
+# 		assert nav_complete_recovery.error_message is None, 'Recovery navigation should succeed'
+
+# 	finally:
+# 		await session.stop()

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_event_ScrollEvent.py

@@ -0,0 +1,409 @@
+import asyncio
+
+import pytest
+from pytest_httpserver import HTTPServer
+
+from browser_use.agent.views import ActionModel, ActionResult
+from browser_use.browser import BrowserSession
+from browser_use.browser.profile import BrowserProfile
+from browser_use.tools.service import Tools
+from browser_use.tools.views import (
+	GoToUrlAction,
+	ScrollAction,
+)
+
+
+@pytest.fixture(scope='session')
+def http_server():
+	"""Create and provide a test HTTP server that serves static content."""
+	server = HTTPServer()
+	server.start()
+
+	# Add routes for common test pages
+	server.expect_request('/').respond_with_data(
+		'<html><head><title>Test Home Page</title></head><body><h1>Test Home Page</h1><p>Welcome to the test site</p></body></html>',
+		content_type='text/html',
+	)
+
+	server.expect_request('/scrollable').respond_with_data(
+		"""
+		<!DOCTYPE html>
+		<html>
+		<head>
+			<title>Scrollable Page</title>
+			<style>
+				body { margin: 0; padding: 20px; }
+				.content { height: 3000px; background: linear-gradient(to bottom, #f0f0f0, #333); }
+				.marker { padding: 20px; background: #007bff; color: white; margin: 500px 0; }
+			</style>
+		</head>
+		<body>
+			<h1>Scrollable Test Page</h1>
+			<div class="content">
+				<div class="marker" id="marker1">Marker 1</div>
+				<div class="marker" id="marker2">Marker 2</div>
+				<div class="marker" id="marker3">Marker 3</div>
+			</div>
+		</body>
+		</html>
+		""",
+		content_type='text/html',
+	)
+
+	yield server
+	server.stop()
+
+
+@pytest.fixture(scope='session')
+def base_url(http_server):
+	"""Return the base URL for the test HTTP server."""
+	return f'http://{http_server.host}:{http_server.port}'
+
+
+@pytest.fixture(scope='module')
+async def browser_session():
+	"""Create and provide a Browser instance with security disabled."""
+	profile = BrowserProfile(headless=True, disable_security=True, cross_origin_iframes=False)
+	session = BrowserSession(browser_profile=profile)
+	await session.start()
+	yield session
+	await session.kill()
+
+
+@pytest.fixture
+def tools():
+	"""Create and provide a Tools instance."""
+	return Tools()
+
+
+class TestScrollActions:
+	"""Test scroll-related actions and events."""
+
+	async def test_scroll_actions(self, tools, browser_session, base_url, http_server):
+		"""Test basic scroll action functionality."""
+
+		# Navigate to scrollable page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/scrollable', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+
+		# Test 1: Basic page scroll down
+		scroll_action = {'scroll': ScrollAction(down=True, num_pages=1.0)}
+
+		class ScrollActionModel(ActionModel):
+			scroll: ScrollAction | None = None
+
+		result = await tools.act(ScrollActionModel(**scroll_action), browser_session)
+
+		# Verify scroll down succeeded
+		assert isinstance(result, ActionResult)
+		assert result.error is None, f'Scroll down failed: {result.error}'
+		assert result.extracted_content is not None
+		assert 'Scrolled down' in result.extracted_content
+		assert 'the page' in result.extracted_content
+
+		# Test 2: Basic page scroll up
+		scroll_up_action = {'scroll': ScrollAction(down=False, num_pages=0.5)}
+		result = await tools.act(ScrollActionModel(**scroll_up_action), browser_session)
+
+		assert isinstance(result, ActionResult)
+		assert result.error is None, f'Scroll up failed: {result.error}'
+		assert result.extracted_content is not None
+		assert 'Scrolled up' in result.extracted_content
+		assert '0.5 pages' in result.extracted_content
+
+		# Test 3: Test with invalid element index (should error)
+		invalid_scroll_action = {'scroll': ScrollAction(down=True, num_pages=1.0, frame_element_index=999)}
+		result = await tools.act(ScrollActionModel(**invalid_scroll_action), browser_session)
+
+		# This should fail with error about element not found
+		assert isinstance(result, ActionResult)
+		assert result.error is not None, 'Expected error for invalid element index'
+		assert 'Element index 999 not found' in result.error or 'Failed to execute scroll' in result.error
+
+		# Test 4: Model parameter validation
+		scroll_with_index = ScrollAction(down=True, num_pages=1.0, frame_element_index=5)
+		assert scroll_with_index.down is True
+		assert scroll_with_index.num_pages == 1.0
+		assert scroll_with_index.frame_element_index == 5
+
+		scroll_without_index = ScrollAction(down=False, num_pages=0.25)
+		assert scroll_without_index.down is False
+		assert scroll_without_index.num_pages == 0.25
+		assert scroll_without_index.frame_element_index is None
+
+	async def test_scroll_with_cross_origin_disabled(self, browser_session, base_url):
+		"""Test that scroll works when cross_origin_iframes is disabled."""
+		from browser_use.browser.events import ScrollEvent
+
+		# Navigate to a page
+		await browser_session._cdp_navigate(f'{base_url}/scrollable')
+		await asyncio.sleep(0.5)
+
+		# Test simple scroll - should not hang
+		event = browser_session.event_bus.dispatch(ScrollEvent(direction='down', amount=500))
+		result = await asyncio.wait_for(event, timeout=3.0)
+		assert result is not None
+
+		# Test scroll up
+		event = browser_session.event_bus.dispatch(ScrollEvent(direction='up', amount=200))
+		result = await asyncio.wait_for(event, timeout=3.0)
+		assert result is not None
+
+	async def test_scroll_non_scrollable_page(self, browser_session, base_url, http_server):
+		"""Test scrolling a page that's only 100px tall (not scrollable)."""
+		from browser_use.browser.events import ScrollEvent
+
+		# Add a non-scrollable page (content fits in viewport)
+		http_server.expect_request('/non-scrollable').respond_with_data(
+			"""
+			<!DOCTYPE html>
+			<html>
+			<head>
+				<title>Non-Scrollable Page</title>
+				<style>
+					body { margin: 0; padding: 10px; height: 80px; overflow: hidden; }
+					.content { height: 60px; background: #f0f0f0; }
+				</style>
+			</head>
+			<body>
+				<div class="content">This page is too small to scroll</div>
+			</body>
+			</html>
+			""",
+			content_type='text/html',
+		)
+
+		# Navigate to non-scrollable page
+		await browser_session._cdp_navigate(f'{base_url}/non-scrollable')
+		await asyncio.sleep(0.5)
+
+		# Get initial scroll position
+		cdp_session = await browser_session.get_or_create_cdp_session()
+		initial_scroll = await browser_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': 'window.pageYOffset', 'returnByValue': True},
+			session_id=cdp_session.session_id,
+		)
+		initial_y = initial_scroll.get('result', {}).get('value', 0)
+
+		# Try to scroll down - should succeed but not actually move
+		event = browser_session.event_bus.dispatch(ScrollEvent(direction='down', amount=500))
+		await event
+		result = await event.event_result(raise_if_any=True, raise_if_none=False)
+		assert result is None
+
+		# Check scroll position didn't change (page isn't scrollable)
+		final_scroll = await browser_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': 'window.pageYOffset', 'returnByValue': True},
+			session_id=cdp_session.session_id,
+		)
+		final_y = final_scroll.get('result', {}).get('value', 0)
+		assert final_y == initial_y, f'Scroll position changed on non-scrollable page: {initial_y} -> {final_y}'
+
+	async def test_scroll_very_long_page(self, browser_session, base_url, http_server):
+		"""Test scrolling a very long page (over 10,000px) by 8,000px."""
+		from browser_use.browser.events import ScrollEvent
+
+		# Add a very long page
+		http_server.expect_request('/very-long').respond_with_data(
+			"""
+			<!DOCTYPE html>
+			<html>
+			<head>
+				<title>Very Long Page</title>
+				<style>
+					body { margin: 0; padding: 20px; }
+					.content { height: 12000px; background: linear-gradient(to bottom, #f0f0f0, #333); }
+					.marker { padding: 20px; background: #007bff; color: white; margin: 2000px 0; }
+				</style>
+			</head>
+			<body>
+				<h1 id="top">Very Long Page - Top</h1>
+				<div class="content">
+					<div class="marker" id="marker1">Marker 1 at 2000px</div>
+					<div class="marker" id="marker2">Marker 2 at 4000px</div>
+					<div class="marker" id="marker3">Marker 3 at 6000px</div>
+					<div class="marker" id="marker4">Marker 4 at 8000px</div>
+					<div class="marker" id="marker5">Marker 5 at 10000px</div>
+				</div>
+				<h1 id="bottom">Very Long Page - Bottom</h1>
+			</body>
+			</html>
+			""",
+			content_type='text/html',
+		)
+
+		# Navigate to very long page
+		await browser_session._cdp_navigate(f'{base_url}/very-long')
+		await asyncio.sleep(0.5)
+
+		# Get initial scroll position
+		cdp_session = await browser_session.get_or_create_cdp_session()
+		initial_scroll = await browser_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': 'window.pageYOffset', 'returnByValue': True},
+			session_id=cdp_session.session_id,
+		)
+		initial_y = initial_scroll.get('result', {}).get('value', 0)
+		assert initial_y == 0, f'Page should start at top, but pageYOffset is {initial_y}'
+
+		# Scroll down by 8000px
+		event = browser_session.event_bus.dispatch(ScrollEvent(direction='down', amount=8000))
+		await event
+		result = await event.event_result(raise_if_any=True, raise_if_none=False)
+		assert result is None  # ScrollEvent does not return a result
+
+		# Wait a bit for scroll to take effect
+		await asyncio.sleep(0.5)
+
+		# Check scroll position moved significantly
+		final_scroll = await browser_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': 'window.pageYOffset', 'returnByValue': True},
+			session_id=cdp_session.session_id,
+		)
+		final_y = final_scroll.get('result', {}).get('value', 0)
+
+		# Get page height to understand constraints
+		page_height = await browser_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': 'document.body.scrollHeight', 'returnByValue': True},
+			session_id=cdp_session.session_id,
+		)
+		scroll_height = page_height.get('result', {}).get('value', 0)
+
+		# Should have scrolled down significantly (might not be exactly 8000 due to viewport constraints)
+		assert final_y > 5000, f'Expected to scroll significantly (page height: {scroll_height}px), but only at {final_y}px'
+
+		# Verify we can see marker 4 which is at 8000px
+		marker4_visible = await browser_session.cdp_client.send.Runtime.evaluate(
+			params={
+				'expression': """
+					(() => {
+						const marker = document.getElementById('marker4');
+						const rect = marker.getBoundingClientRect();
+						return rect.top >= 0 && rect.top <= window.innerHeight;
+					})()
+				""",
+				'returnByValue': True,
+			},
+			session_id=cdp_session.session_id,
+		)
+		assert marker4_visible.get('result', {}).get('value', False), 'Marker 4 should be visible after scrolling 8000px'
+
+	async def test_scroll_iframe_content(self, browser_session, base_url, http_server):
+		"""Test scrolling inside a same-origin iframe."""
+		from browser_use.browser.events import ScrollEvent
+
+		# Add iframe content page
+		http_server.expect_request('/iframe-content').respond_with_data(
+			"""
+			<!DOCTYPE html>
+			<html>
+			<head>
+				<style>
+					body { margin: 0; padding: 10px; }
+					.content { height: 2000px; background: linear-gradient(to bottom, #e0e0e0, #666); }
+				</style>
+			</head>
+			<body>
+				<h2 id="iframe-top">Iframe Content - Top</h2>
+				<div class="content">
+					<div style="margin-top: 900px;">Middle of iframe content</div>
+					<div style="margin-top: 900px;">Bottom of iframe content</div>
+				</div>
+			</body>
+			</html>
+			""",
+			content_type='text/html',
+		)
+
+		# Add main page with iframe
+		http_server.expect_request('/page-with-iframe').respond_with_data(
+			f"""
+			<!DOCTYPE html>
+			<html>
+			<head>
+				<title>Page with Iframe</title>
+				<style>
+					body {{ margin: 0; padding: 20px; }}
+					#main-content {{ height: 200px; background: #f0f0f0; }}
+					#scrollable-iframe {{ 
+						width: 100%; 
+						height: 400px; 
+						border: 2px solid #333;
+					}}
+				</style>
+			</head>
+			<body>
+				<div id="main-content">
+					<h1>Main Page Content</h1>
+					<p>This is the main page with an embedded iframe below.</p>
+				</div>
+				<iframe id="scrollable-iframe" src="{base_url}/iframe-content"></iframe>
+				<div style="height: 200px; background: #e0e0e0;">
+					<p>Content after iframe</p>
+				</div>
+			</body>
+			</html>
+			""",
+			content_type='text/html',
+		)
+
+		# Navigate to page with iframe
+		await browser_session._cdp_navigate(f'{base_url}/page-with-iframe')
+		await asyncio.sleep(1.0)  # Give iframe time to load
+
+		# Get initial scroll position of main page and iframe
+		cdp_session = await browser_session.get_or_create_cdp_session()
+
+		# Check main page scroll
+		main_scroll = await browser_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': 'window.pageYOffset', 'returnByValue': True},
+			session_id=cdp_session.session_id,
+		)
+		main_y = main_scroll.get('result', {}).get('value', 0)
+
+		# Check iframe scroll (should start at 0)
+		iframe_initial = await browser_session.cdp_client.send.Runtime.evaluate(
+			params={
+				'expression': """
+					(() => {
+						const iframe = document.getElementById('scrollable-iframe');
+						if (iframe && iframe.contentWindow) {
+							return iframe.contentWindow.pageYOffset || 0;
+						}
+						return -1;
+					})()
+				""",
+				'returnByValue': True,
+			},
+			session_id=cdp_session.session_id,
+		)
+		iframe_y = iframe_initial.get('result', {}).get('value', -1)
+		assert iframe_y == 0, f'Iframe should start at top, but pageYOffset is {iframe_y}'
+
+		# Scroll the main page first to bring iframe into view
+		event = browser_session.event_bus.dispatch(ScrollEvent(direction='down', amount=100))
+		await asyncio.wait_for(event, timeout=3.0)
+
+		# Now try to scroll inside the iframe
+		# Note: This would require finding the iframe element and scrolling it specifically
+		# For now, we just verify the iframe exists and is scrollable
+		iframe_scrollable = await browser_session.cdp_client.send.Runtime.evaluate(
+			params={
+				'expression': """
+					(() => {
+						const iframe = document.getElementById('scrollable-iframe');
+						if (iframe && iframe.contentDocument) {
+							const iframeBody = iframe.contentDocument.body;
+							return iframeBody.scrollHeight > iframe.clientHeight;
+						}
+						return false;
+					})()
+				""",
+				'returnByValue': True,
+			},
+			session_id=cdp_session.session_id,
+		)
+		assert iframe_scrollable.get('result', {}).get('value', False), 'Iframe should be scrollable'

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_profile_disable_security.py

@@ -0,0 +1,83 @@
+"""Test that disable_security flag properly merges --disable-features flags without breaking extensions."""
+
+import tempfile
+
+from browser_use.browser.profile import BrowserProfile
+
+
+class TestBrowserProfileDisableSecurity:
+	"""Test disable_security flag behavior."""
+
+	def test_disable_security_preserves_extension_features(self):
+		"""Test that disable_security=True doesn't break extension features by properly merging --disable-features flags."""
+
+		# Test with disable_security=False (baseline)
+		profile_normal = BrowserProfile(disable_security=False, user_data_dir=tempfile.mkdtemp(prefix='test-normal-'))
+		profile_normal.detect_display_configuration()
+		args_normal = profile_normal.get_args()
+
+		# Test with disable_security=True
+		profile_security_disabled = BrowserProfile(disable_security=True, user_data_dir=tempfile.mkdtemp(prefix='test-security-'))
+		profile_security_disabled.detect_display_configuration()
+		args_security_disabled = profile_security_disabled.get_args()
+
+		# Extract disable-features args
+		def extract_disable_features(args):
+			for arg in args:
+				if arg.startswith('--disable-features='):
+					return set(arg.split('=', 1)[1].split(','))
+			return set()
+
+		features_normal = extract_disable_features(args_normal)
+		features_security_disabled = extract_disable_features(args_security_disabled)
+
+		# Check that extension-related features are preserved
+		extension_features = {
+			'ExtensionManifestV2Disabled',
+			'ExtensionDisableUnsupportedDeveloper',
+			'ExtensionManifestV2Unsupported',
+		}
+
+		security_features = {'IsolateOrigins', 'site-per-process'}
+
+		# Verify that security disabled has both extension and security features
+		missing_extension_features = extension_features - features_security_disabled
+		missing_security_features = security_features - features_security_disabled
+
+		assert not missing_extension_features, (
+			f'Missing extension features when disable_security=True: {missing_extension_features}'
+		)
+		assert not missing_security_features, f'Missing security features when disable_security=True: {missing_security_features}'
+
+		# Verify that security disabled profile has more features than normal (due to added security features)
+		assert len(features_security_disabled) > len(features_normal), (
+			'Security disabled profile should have more features than normal profile'
+		)
+
+		# Verify all normal features are preserved in security disabled profile
+		missing_normal_features = features_normal - features_security_disabled
+		assert not missing_normal_features, f'Normal features missing from security disabled profile: {missing_normal_features}'
+
+	def test_disable_features_flag_deduplication(self):
+		"""Test that duplicate --disable-features values are properly deduplicated."""
+
+		profile = BrowserProfile(
+			disable_security=True,
+			user_data_dir=tempfile.mkdtemp(prefix='test-dedup-'),
+			# Add duplicate features to test deduplication
+			args=['--disable-features=TestFeature1,TestFeature2', '--disable-features=TestFeature2,TestFeature3'],
+		)
+		profile.detect_display_configuration()
+		args = profile.get_args()
+
+		# Extract disable-features args
+		disable_features_args = [arg for arg in args if arg.startswith('--disable-features=')]
+
+		# Should only have one consolidated --disable-features flag
+		assert len(disable_features_args) == 1, f'Expected 1 disable-features flag, got {len(disable_features_args)}'
+
+		features = set(disable_features_args[0].split('=', 1)[1].split(','))
+
+		# Should have all test features without duplicates
+		expected_test_features = {'TestFeature1', 'TestFeature2', 'TestFeature3'}
+		assert expected_test_features.issubset(features), f'Missing test features: {expected_test_features - features}'

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_session_element_cache.py

@@ -0,0 +1,242 @@
+"""
+Systematic debugging of the selector map issue.
+Test each assumption step by step to isolate the problem.
+"""
+
+import pytest
+
+from browser_use.browser import BrowserSession
+from browser_use.browser.profile import BrowserProfile
+from browser_use.tools.service import Tools
+
+
+@pytest.fixture
+def httpserver(make_httpserver):
+	"""Create and provide a test HTTP server that serves static content."""
+	server = make_httpserver
+
+	# Add routes for test pages
+	server.expect_request('/').respond_with_data(
+		"""<html>
+		<head><title>Test Home Page</title></head>
+		<body>
+			<h1>Test Home Page</h1>
+			<a href="/page1" id="link1">Link 1</a>
+			<button id="button1">Button 1</button>
+			<input type="text" id="input1" />
+			<div id="div1" class="clickable">Clickable Div</div>
+		</body>
+		</html>""",
+		content_type='text/html',
+	)
+
+	server.expect_request('/page1').respond_with_data(
+		"""<html>
+		<head><title>Test Page 1</title></head>
+		<body>
+			<h1>Test Page 1</h1>
+			<p>This is test page 1</p>
+			<a href="/">Back to home</a>
+		</body>
+		</html>""",
+		content_type='text/html',
+	)
+
+	server.expect_request('/simple').respond_with_data(
+		"""<html>
+		<head><title>Simple Page</title></head>
+		<body>
+			<h1>Simple Page</h1>
+			<p>This is a simple test page</p>
+			<a href="/">Home</a>
+		</body>
+		</html>""",
+		content_type='text/html',
+	)
+
+	return server
+
+
+@pytest.fixture
+async def browser_session():
+	"""Create a real browser session for testing."""
+	session = BrowserSession(
+		browser_profile=BrowserProfile(
+			user_data_dir=None,  # Use temporary profile
+			headless=True,
+		)
+	)
+	await session.start()
+	yield session
+	await session.stop()
+
+
+@pytest.fixture
+def tools():
+	"""Create a tools instance."""
+	return Tools()
+
+
+@pytest.mark.asyncio
+async def test_assumption_1_dom_processing_works(browser_session, httpserver):
+	"""Test assumption 1: DOM processing works and finds elements."""
+	# Go to a simple page using CDP events
+	from browser_use.browser.events import NavigateToUrlEvent
+
+	event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))
+	await event
+	await event.event_result(raise_if_any=True, raise_if_none=False)
+
+	# Trigger DOM processing
+	state = await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=False)
+
+	print('DOM processing result:')
+	print(f'  - Elements found: {len(state.dom_state.selector_map)}')
+	print(f'  - Element indices: {list(state.dom_state.selector_map.keys())}')
+
+	# Verify DOM processing works
+	assert len(state.dom_state.selector_map) > 0, 'DOM processing should find interactive elements'
+
+
+@pytest.mark.asyncio
+async def test_assumption_2_cached_selector_map_persists(browser_session, httpserver):
+	"""Test assumption 2: Cached selector map persists after get_state_summary."""
+	# Go to a simple page using CDP events
+	from browser_use.browser.events import NavigateToUrlEvent
+
+	event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))
+	await event
+	await event.event_result(raise_if_any=True, raise_if_none=False)
+
+	# Trigger DOM processing and cache
+	state = await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=False)
+	initial_selector_map = dict(state.dom_state.selector_map)
+
+	# Check if cached selector map is still available
+	cached_selector_map = await browser_session.get_selector_map()
+
+	print('Selector map persistence:')
+	print(f'  - Initial elements: {len(initial_selector_map)}')
+	print(f'  - Cached elements: {len(cached_selector_map)}')
+	print(f'  - Maps are identical: {initial_selector_map.keys() == cached_selector_map.keys()}')
+
+	# Verify the cached map persists
+	assert len(cached_selector_map) > 0, 'Cached selector map should persist'
+	assert initial_selector_map.keys() == cached_selector_map.keys(), 'Cached map should match initial map'
+
+
+@pytest.mark.asyncio
+async def test_assumption_3_action_gets_same_selector_map(browser_session, tools, httpserver):
+	"""Test assumption 3: Action gets the same selector map as cached."""
+	# Go to a simple page using CDP events
+	from browser_use.browser.events import NavigateToUrlEvent
+
+	event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))
+	await event
+	await event.event_result(raise_if_any=True, raise_if_none=False)
+
+	# Trigger DOM processing and cache
+	await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=False)
+	cached_selector_map = await browser_session.get_selector_map()
+
+	print('Pre-action state:')
+	print(f'  - Cached elements: {len(cached_selector_map)}')
+	print(f'  - Element 0 exists in cache: {0 in cached_selector_map}')
+
+	# Create a test action that checks the selector map it receives
+	@tools.registry.action('Test: Check selector map')
+	async def test_check_selector_map(browser_session: BrowserSession):
+		from browser_use import ActionResult
+
+		action_selector_map = await browser_session.get_selector_map()
+		return ActionResult(
+			extracted_content=f'Action sees {len(action_selector_map)} elements, index 0 exists: {0 in action_selector_map}',
+			include_in_memory=False,
+		)
+
+	# Execute the test action
+	result = await tools.registry.execute_action('test_check_selector_map', {}, browser_session=browser_session)
+
+	print(f'Action result: {result.extracted_content}')
+
+	# Verify the action sees the same selector map
+	assert 'index 0 exists: False' in result.extracted_content, 'Element 0 should not exist (elements start at 1)'
+
+
+@pytest.mark.asyncio
+async def test_assumption_4_click_action_specific_issue(browser_session, tools, httpserver):
+	"""Test assumption 4: Specific issue with click_element_by_index action."""
+	# Go to a simple page using CDP events
+	from browser_use.browser.events import NavigateToUrlEvent
+
+	event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))
+	await event
+	await event.event_result(raise_if_any=True, raise_if_none=False)
+
+	# Trigger DOM processing and cache
+	await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=False)
+	cached_selector_map = await browser_session.get_selector_map()
+
+	print('Pre-click state:')
+	print(f'  - Cached elements: {len(cached_selector_map)}')
+	print(f'  - Element 0 exists: {0 in cached_selector_map}')
+
+	# Create a test action that replicates click_element_by_index logic
+	@tools.registry.action('Test: Debug click logic')
+	async def test_debug_click_logic(index: int, browser_session: BrowserSession):
+		from browser_use import ActionResult
+
+		# This is the exact logic from click_element_by_index
+		selector_map = await browser_session.get_selector_map()
+
+		print(f'  - Action selector map size: {len(selector_map)}')
+		print(f'  - Action selector map keys: {list(selector_map.keys())[:10]}')  # First 10
+		print(f'  - Index {index} in selector map: {index in selector_map}')
+
+		if index not in selector_map:
+			return ActionResult(
+				error=f'Debug: Element with index {index} does not exist in map of size {len(selector_map)}',
+				include_in_memory=False,
+			)
+
+		return ActionResult(
+			extracted_content=f'Debug: Element {index} found in map of size {len(selector_map)}', include_in_memory=False
+		)
+
+	# Test with index 1 (elements start at 1, not 0)
+	result = await tools.registry.execute_action('test_debug_click_logic', {'index': 1}, browser_session=browser_session)
+
+	print(f'Debug click result: {result.extracted_content or result.error}')
+
+	# This will help us see exactly what the click action sees
+	if result.error:
+		pytest.fail(f'Click logic debug failed: {result.error}')
+
+
+@pytest.mark.asyncio
+async def test_assumption_5_multiple_get_selector_map_calls(browser_session, httpserver):
+	"""Test assumption 5: Multiple calls to get_selector_map return consistent results."""
+	# Go to a simple page using CDP events
+	from browser_use.browser.events import NavigateToUrlEvent
+
+	event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))
+	await event
+	await event.event_result(raise_if_any=True, raise_if_none=False)
+
+	# Trigger DOM processing and cache
+	await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=False)
+
+	# Call get_selector_map multiple times
+	map1 = await browser_session.get_selector_map()
+	map2 = await browser_session.get_selector_map()
+	map3 = await browser_session.get_selector_map()
+
+	print('Multiple selector map calls:')
+	print(f'  - Call 1: {len(map1)} elements')
+	print(f'  - Call 2: {len(map2)} elements')
+	print(f'  - Call 3: {len(map3)} elements')
+	print(f'  - All calls identical: {map1.keys() == map2.keys() == map3.keys()}')
+
+	# Verify consistency
+	assert len(map1) == len(map2) == len(map3), 'Multiple calls should return same size'
+	assert map1.keys() == map2.keys() == map3.keys(), 'Multiple calls should return same elements'

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_session_output_paths.py

@@ -0,0 +1,219 @@
+"""Test all recording and save functionality for Agent and BrowserSession."""
+
+from pathlib import Path
+
+import pytest
+
+from browser_use import Agent, AgentHistoryList
+from browser_use.browser import BrowserProfile, BrowserSession
+from tests.ci.conftest import create_mock_llm
+
+
+@pytest.fixture
+def test_dir(tmp_path):
+	"""Create a test directory that gets cleaned up after each test."""
+	test_path = tmp_path / 'test_recordings'
+	test_path.mkdir(exist_ok=True)
+	yield test_path
+
+
+@pytest.fixture
+async def httpserver_url(httpserver):
+	"""Simple test page."""
+	# Use expect_ordered_request with multiple handlers to handle repeated requests
+	for _ in range(10):  # Allow up to 10 requests to the same URL
+		httpserver.expect_ordered_request('/').respond_with_data(
+			"""
+			<!DOCTYPE html>
+			<html>
+			<head>
+				<title>Test Page</title>
+			</head>
+			<body>
+				<h1>Test Recording Page</h1>
+				<input type="text" id="search" placeholder="Search here" />
+				<button type="button" id="submit">Submit</button>
+			</body>
+			</html>
+			""",
+			content_type='text/html',
+		)
+	return httpserver.url_for('/')
+
+
+@pytest.fixture
+def llm():
+	"""Create mocked LLM instance for tests."""
+	return create_mock_llm()
+
+
+@pytest.fixture
+def interactive_llm(httpserver_url):
+	"""Create mocked LLM that navigates to page and interacts with elements."""
+	actions = [
+		# First action: Navigate to the page
+		f"""
+		{{
+			"thinking": "null",
+			"evaluation_previous_goal": "Starting the task",
+			"memory": "Need to navigate to the test page",
+			"next_goal": "Navigate to the URL",
+			"action": [
+				{{
+					"go_to_url": {{
+						"url": "{httpserver_url}",
+						"new_tab": false
+					}}
+				}}
+			]
+		}}
+		""",
+		# Second action: Click in the search box
+		"""
+		{
+			"thinking": "null",
+			"evaluation_previous_goal": "Successfully navigated to the page",
+			"memory": "Page loaded, can see search box and submit button",
+			"next_goal": "Click on the search box to focus it",
+			"action": [
+				{
+					"click_element_by_index": {
+						"index": 0
+					}
+				}
+			]
+		}
+		""",
+		# Third action: Type text in the search box
+		"""
+		{
+			"thinking": "null",
+			"evaluation_previous_goal": "Clicked on search box",
+			"memory": "Search box is focused and ready for input",
+			"next_goal": "Type 'test' in the search box",
+			"action": [
+				{
+					"input_text": {
+						"index": 0,
+						"text": "test"
+					}
+				}
+			]
+		}
+		""",
+		# Fourth action: Click submit button
+		"""
+		{
+			"thinking": "null",
+			"evaluation_previous_goal": "Typed 'test' in search box",
+			"memory": "Text 'test' has been entered successfully",
+			"next_goal": "Click the submit button to complete the task",
+			"action": [
+				{
+					"click_element_by_index": {
+						"index": 1
+					}
+				}
+			]
+		}
+		""",
+		# Fifth action: Done - task completed
+		"""
+		{
+			"thinking": "null",
+			"evaluation_previous_goal": "Clicked the submit button",
+			"memory": "Successfully navigated to the page, typed 'test' in the search box, and clicked submit",
+			"next_goal": "Task completed",
+			"action": [
+				{
+					"done": {
+						"text": "Task completed - typed 'test' in search box and clicked submit",
+						"success": true
+					}
+				}
+			]
+		}
+		""",
+	]
+	return create_mock_llm(actions)
+
+
+class TestAgentRecordings:
+	"""Test Agent save_conversation_path and generate_gif parameters."""
+
+	@pytest.mark.parametrize('path_type', ['with_slash', 'without_slash', 'deep_directory'])
+	async def test_save_conversation_path(self, test_dir, httpserver_url, llm, path_type):
+		"""Test saving conversation with different path types."""
+		if path_type == 'with_slash':
+			conversation_path = test_dir / 'logs' / 'conversation'
+		elif path_type == 'without_slash':
+			conversation_path = test_dir / 'logs'
+		else:  # deep_directory
+			conversation_path = test_dir / 'logs' / 'deep' / 'directory' / 'conversation'
+
+		browser_session = BrowserSession(browser_profile=BrowserProfile(headless=True, disable_security=True, user_data_dir=None))
+		await browser_session.start()
+		try:
+			agent = Agent(
+				task=f'go to {httpserver_url} and type "test" in the search box',
+				llm=llm,
+				browser_session=browser_session,
+				save_conversation_path=str(conversation_path),
+			)
+			history: AgentHistoryList = await agent.run(max_steps=2)
+
+			result = history.final_result()
+			assert result is not None
+
+			# Check that the conversation directory and files were created
+			assert conversation_path.exists(), f'{path_type}: conversation directory was not created'
+			# Files are now always created as conversation_<agent_id>_<step>.txt inside the directory
+			conversation_files = list(conversation_path.glob('conversation_*.txt'))
+			assert len(conversation_files) > 0, f'{path_type}: conversation file was not created in {conversation_path}'
+		finally:
+			await browser_session.kill()
+
+	@pytest.mark.skip(reason='TODO: fix')
+	@pytest.mark.parametrize('generate_gif', [False, True, 'custom_path'])
+	async def test_generate_gif(self, test_dir, httpserver_url, llm, generate_gif):
+		"""Test GIF generation with different settings."""
+		# Clean up any existing GIFs first
+		for gif in Path.cwd().glob('agent_*.gif'):
+			gif.unlink()
+
+		gif_param = generate_gif
+		expected_gif_path = None
+
+		if generate_gif == 'custom_path':
+			expected_gif_path = test_dir / 'custom_agent.gif'
+			gif_param = str(expected_gif_path)
+
+		browser_session = BrowserSession(browser_profile=BrowserProfile(headless=True, disable_security=True, user_data_dir=None))
+		await browser_session.start()
+		try:
+			agent = Agent(
+				task=f'go to {httpserver_url}',
+				llm=llm,
+				browser_session=browser_session,
+				generate_gif=gif_param,
+			)
+			history: AgentHistoryList = await agent.run(max_steps=2)
+
+			result = history.final_result()
+			assert result is not None
+
+			# Check GIF creation
+			if generate_gif is False:
+				gif_files = list(Path.cwd().glob('*.gif'))
+				assert len(gif_files) == 0, 'GIF file was created when generate_gif=False'
+			elif generate_gif is True:
+				# With mock LLM that doesn't navigate, all screenshots will be about:blank placeholders
+				# So no GIF will be created (this is expected behavior)
+				gif_files = list(Path.cwd().glob('agent_history.gif'))
+				assert len(gif_files) == 0, 'GIF should not be created when all screenshots are placeholders'
+			else:  # custom_path
+				assert expected_gif_path is not None, 'expected_gif_path should be set for custom_path'
+				# With mock LLM that doesn't navigate, no GIF will be created
+				assert not expected_gif_path.exists(), 'GIF should not be created when all screenshots are placeholders'
+		finally:
+			await browser_session.kill()

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_session_proxy.py

@@ -0,0 +1,113 @@
+import asyncio
+from typing import Any
+
+import pytest
+
+from browser_use.browser import BrowserProfile, BrowserSession
+from browser_use.browser.profile import ProxySettings
+from browser_use.config import CONFIG
+
+
+def test_chromium_args_include_proxy_flags():
+	profile = BrowserProfile(
+		headless=True,
+		user_data_dir=str(CONFIG.BROWSER_USE_PROFILES_DIR / 'proxy-smoke'),
+		proxy=ProxySettings(
+			server='http://proxy.local:4242',
+			bypass='localhost,127.0.0.1',
+		),
+	)
+	args = profile.get_args()
+	assert any(a == '--proxy-server=http://proxy.local:4242' for a in args), args
+	assert any(a == '--proxy-bypass-list=localhost,127.0.0.1' for a in args), args
+
+
+@pytest.mark.asyncio
+async def test_cdp_proxy_auth_handler_registers_and_responds():
+	# Create profile with proxy auth credentials
+	profile = BrowserProfile(
+		headless=True,
+		user_data_dir=str(CONFIG.BROWSER_USE_PROFILES_DIR / 'proxy-smoke'),
+		proxy=ProxySettings(username='user', password='pass'),
+	)
+	session = BrowserSession(browser_profile=profile)
+
+	# Stub CDP client with minimal Fetch support
+	class StubCDP:
+		def __init__(self) -> None:
+			self.enabled = False
+			self.last_auth: dict[str, Any] | None = None
+			self.last_default: dict[str, Any] | None = None
+			self.auth_callback = None
+			self.request_paused_callback = None
+
+			class _FetchSend:
+				def __init__(self, outer: 'StubCDP') -> None:
+					self._outer = outer
+
+				async def enable(self, params: dict, session_id: str | None = None) -> None:
+					self._outer.enabled = True
+
+				async def continueWithAuth(self, params: dict, session_id: str | None = None) -> None:
+					self._outer.last_auth = {'params': params, 'session_id': session_id}
+
+				async def continueRequest(self, params: dict, session_id: str | None = None) -> None:
+					# no-op; included to mirror CDP API surface used by impl
+					pass
+
+			class _Send:
+				def __init__(self, outer: 'StubCDP') -> None:
+					self.Fetch = _FetchSend(outer)
+
+			class _FetchRegister:
+				def __init__(self, outer: 'StubCDP') -> None:
+					self._outer = outer
+
+				def authRequired(self, callback) -> None:
+					self._outer.auth_callback = callback
+
+				def requestPaused(self, callback) -> None:
+					self._outer.request_paused_callback = callback
+
+			class _Register:
+				def __init__(self, outer: 'StubCDP') -> None:
+					self.Fetch = _FetchRegister(outer)
+
+			self.send = _Send(self)
+			self.register = _Register(self)
+
+	root = StubCDP()
+
+	# Attach stubs to session
+	session._cdp_client_root = root  # type: ignore[attr-defined]
+	# No need to attach a real CDPSession; _setup_proxy_auth works with root client
+
+	# Should register Fetch handler and enable auth handling without raising
+	await session._setup_proxy_auth()
+
+	assert root.enabled is True
+	assert callable(root.auth_callback)
+
+	# Simulate proxy auth required event
+	ev = {'requestId': 'r1', 'authChallenge': {'source': 'Proxy'}}
+	root.auth_callback(ev, session_id='s1')  # type: ignore[misc]
+
+	# Let scheduled task run
+	await asyncio.sleep(0.05)
+
+	assert root.last_auth is not None
+	params = root.last_auth['params']
+	assert params['authChallengeResponse']['response'] == 'ProvideCredentials'
+	assert params['authChallengeResponse']['username'] == 'user'
+	assert params['authChallengeResponse']['password'] == 'pass'
+	assert root.last_auth['session_id'] == 's1'
+
+	# Now simulate a non-proxy auth challenge and ensure default handling
+	ev2 = {'requestId': 'r2', 'authChallenge': {'source': 'Server'}}
+	root.auth_callback(ev2, session_id='s2')  # type: ignore[misc]
+	await asyncio.sleep(0.05)
+	# After non-proxy challenge, last_auth should reflect Default response
+	assert root.last_auth is not None
+	params2 = root.last_auth['params']
+	assert params2['requestId'] == 'r2'
+	assert params2['authChallengeResponse']['response'] == 'Default'

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_session_recent_events.py

@@ -0,0 +1,355 @@
+"""
+Test browser session recent events tracking functionality.
+"""
+
+import json
+import time
+
+import pytest
+from pytest_httpserver import HTTPServer
+from werkzeug.wrappers import Response
+
+from browser_use.browser import BrowserProfile, BrowserSession
+from browser_use.browser.events import NavigateToUrlEvent, ScreenshotEvent
+
+
+class TestBrowserRecentEvents:
+	"""Test recent events tracking functionality"""
+
+	async def test_recent_events_on_successful_load(self, httpserver: HTTPServer):
+		"""Test that recent events shows successful navigation when page loads successfully"""
+		# Set up a simple page that loads quickly
+		httpserver.expect_request('/fast').respond_with_data(
+			'<html><head><title>Fast Page</title></head><body><h1>Quick loading page</h1></body></html>',
+			content_type='text/html',
+		)
+
+		browser_session = BrowserSession(
+			browser_profile=BrowserProfile(
+				headless=True,
+				user_data_dir=None,
+				keep_alive=False,
+			)
+		)
+
+		try:
+			await browser_session.start()
+
+			# Navigate to the fast-loading page
+			event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/fast')))
+			await event
+			await event.event_result(raise_if_any=True, raise_if_none=False)
+
+			# Get browser state with recent events
+			state = await browser_session.get_browser_state_summary(include_recent_events=True)
+
+			# Recent events should show successful navigation
+			assert state.recent_events is not None
+
+			# Parse JSON and verify events
+			events = json.loads(state.recent_events)
+			event_types = [e.get('event_type') for e in events]
+
+			# Should have navigation events
+			assert 'NavigationCompleteEvent' in event_types, 'Should have NavigationCompleteEvent'
+
+			# Check the navigation was successful (no errors)
+			nav_events = [e for e in events if e.get('event_type') == 'NavigationCompleteEvent']
+			last_nav = nav_events[-1]
+			assert last_nav.get('error_message') is None, 'Should not have error message'
+			# Note: CDP doesn't provide HTTP status directly, so skip status check
+
+		finally:
+			await browser_session.kill()
+
+	# async def test_recent_events_tracks_multiple_navigations(self, httpserver: HTTPServer):
+	# 	"""Test that recent events properly tracks multiple navigations"""
+	# 	# Set up pages
+	# 	slow_html = """
+	# 	<html>
+	# 	<head>
+	# 		<title>Slow Page</title>
+	# 		<script src="/slow.js"></script>
+	# 	</head>
+	# 	<body><h1>Slow page</h1></body>
+	# 	</html>
+	# 	"""
+
+	# 	httpserver.expect_request('/slow').respond_with_data(slow_html, content_type='text/html')
+
+	# 	def slow_handler(req):
+	# 		time.sleep(5)
+	# 		return Response('slow')
+
+	# 	httpserver.expect_request('/slow.js').respond_with_handler(slow_handler)
+
+	# 	httpserver.expect_request('/fast').respond_with_data(
+	# 		'<html><head><title>Fast Page</title></head><body><h1>Fast page</h1></body></html>',
+	# 		content_type='text/html',
+	# 	)
+
+	# 	browser_session = BrowserSession(
+	# 		browser_profile=BrowserProfile(
+	# 			headless=True,
+	# 			user_data_dir=None,
+	# 			keep_alive=False,
+	# 		)
+	# 	)
+
+	# 	try:
+	# 		await browser_session.start()
+
+	# 		# Navigate to slow page
+	# 		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/slow')))
+	# 		await event
+	# 		await event.event_result(raise_if_any=True, raise_if_none=False)
+	# 		state1 = await browser_session.get_browser_state_summary(include_recent_events=True)
+	# 		assert state1.recent_events is not None
+	# 		events1 = json.loads(state1.recent_events)
+	# 		event_types1 = [e.get('event_type') for e in events1]
+	# 		assert 'NavigationCompleteEvent' in event_types1 or 'NavigateToUrlEvent' in event_types1
+
+	# 		# Navigate to fast page
+	# 		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/fast')))
+	# 		await event
+	# 		await event.event_result(raise_if_any=True, raise_if_none=False)
+	# 		state2 = await browser_session.get_browser_state_summary()
+
+	# 		# Recent events should show both navigations
+	# 		assert state2.recent_events is not None
+	# 		events2 = json.loads(state2.recent_events)
+
+	# 		# Count navigation events (last 10 events should include both)
+	# 		nav_complete_count = sum(1 for e in events2 if e.get('event_type') == 'NavigationCompleteEvent')
+	# 		nav_url_count = sum(1 for e in events2 if e.get('event_type') == 'NavigateToUrlEvent')
+
+	# 		# Should have events from both navigations
+	# 		assert nav_complete_count >= 1 or nav_url_count >= 2, 'Recent events should show multiple navigation attempts'
+
+	# 	finally:
+	# 		await browser_session.kill()
+
+	async def test_recent_events_preserved_in_minimal_state(self, httpserver: HTTPServer):
+		"""Test that recent events is preserved even when falling back to minimal state"""
+		# Create a page that causes DOM processing to fail
+		malformed_html = """
+		<html>
+		<head>
+			<title>Malformed Page</title>
+			<script src="/slow.js"></script>
+			<script>
+				// This might cause DOM processing issues
+				Object.defineProperty(document, 'querySelectorAll', {
+					get() { throw new Error('DOM processing blocked'); }
+				});
+			</script>
+		</head>
+		<body><h1>Page with DOM issues</h1></body>
+		</html>
+		"""
+
+		httpserver.expect_request('/malformed').respond_with_data(malformed_html, content_type='text/html')
+
+		def slow_handler(req):
+			time.sleep(5)
+			return Response('slow')
+
+		httpserver.expect_request('/slow.js').respond_with_handler(slow_handler)
+
+		browser_session = BrowserSession(
+			browser_profile=BrowserProfile(
+				headless=True,
+				user_data_dir=None,
+				keep_alive=False,
+			)
+		)
+
+		try:
+			await browser_session.start()
+
+			# Navigate to the malformed page
+			event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/malformed')))
+			await event
+			await event.event_result(raise_if_any=True, raise_if_none=False)
+
+			# Get browser state - this might fall back to minimal state
+			state = await browser_session.get_browser_state_summary(include_recent_events=True)
+
+			# Even if we get minimal state, recent events should be preserved
+			assert state.recent_events is not None
+			events = json.loads(state.recent_events)
+			assert len(events) > 0, 'Should have events even in minimal state'
+
+			# Should have navigation attempt
+			event_types = [e.get('event_type') for e in events]
+			assert 'NavigateToUrlEvent' in event_types or 'NavigationCompleteEvent' in event_types, (
+				'Should have navigation events even in minimal state'
+			)
+
+		finally:
+			await browser_session.kill()
+
+	@pytest.mark.parametrize('timeout_seconds', [0.5, 1.0, 2.0])
+	async def test_recent_events_with_different_timeouts(self, httpserver: HTTPServer, timeout_seconds: float):
+		"""Test that recent events captures navigation with different timeout configurations"""
+		# Set up a slow page
+		httpserver.expect_request(f'/timeout_{timeout_seconds}').respond_with_data(
+			f'<html><head><title>Timeout Test {timeout_seconds}s</title>'
+			f'<script src="/slow_{timeout_seconds}.js"></script></head>'
+			f'<body><h1>Testing {timeout_seconds}s timeout</h1></body></html>',
+			content_type='text/html',
+		)
+
+		def very_slow_handler(req):
+			time.sleep(10)
+			return Response('slow')
+
+		httpserver.expect_request(f'/slow_{timeout_seconds}.js').respond_with_handler(very_slow_handler)
+
+		browser_session = BrowserSession(
+			browser_profile=BrowserProfile(
+				headless=True,
+				user_data_dir=None,
+				keep_alive=False,
+				wait_for_network_idle_page_load_time=0.1,
+				minimum_wait_page_load_time=0.1,
+			)
+		)
+
+		try:
+			await browser_session.start()
+
+			# Navigate to the page
+			event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for(f'/timeout_{timeout_seconds}')))
+			await event
+			await event.event_result(raise_if_any=True, raise_if_none=False)
+
+			# Get browser state with recent events
+			state = await browser_session.get_browser_state_summary(include_recent_events=True)
+
+			# Verify recent events captured the navigation
+			assert state.recent_events is not None
+			events = json.loads(state.recent_events)
+			event_types = [e.get('event_type') for e in events]
+
+			# Should have navigation events regardless of timeout
+			assert 'NavigateToUrlEvent' in event_types or 'NavigationCompleteEvent' in event_types, (
+				f'Should have navigation events for {timeout_seconds}s timeout'
+			)
+
+			# If navigation completed with error, it might mention timeout
+			nav_complete_events = [e for e in events if e.get('event_type') == 'NavigationCompleteEvent']
+			if nav_complete_events and nav_complete_events[-1].get('error_message'):
+				print(f'Navigation error for {timeout_seconds}s timeout: {nav_complete_events[-1].get("error_message")}')
+
+		finally:
+			await browser_session.kill()
+
+
+class TestEventHistoryInfrastructure:
+	"""Tests for NEW event history tracking infrastructure only."""
+
+	async def test_event_bus_history_tracking(self, httpserver: HTTPServer):
+		"""Test that event bus properly tracks event history."""
+		browser_session = BrowserSession(browser_profile=BrowserProfile(headless=True, user_data_dir=None, keep_alive=False))
+
+		try:
+			await browser_session.start()
+			initial_history_count = len(browser_session.event_bus.event_history)
+
+			# Set up test page
+			httpserver.expect_request('/history-test').respond_with_data(
+				'<html><body><h1>Event History Test</h1></body></html>',
+				content_type='text/html',
+			)
+
+			# Perform actions that generate events
+			event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/history-test')))
+			await event
+			await event.event_result(raise_if_any=True, raise_if_none=False)
+			screenshot_event = browser_session.event_bus.dispatch(ScreenshotEvent())
+			await screenshot_event
+			await screenshot_event.event_result(raise_if_any=True, raise_if_none=False)
+
+			# Verify event history has grown
+			final_history_count = len(browser_session.event_bus.event_history)
+			assert final_history_count > initial_history_count, 'Event history should track new events'
+
+			# Verify events are stored properly
+			for event_id, event in browser_session.event_bus.event_history.items():
+				assert event_id is not None
+				assert event is not None
+				assert hasattr(event, 'event_type') or hasattr(event, '__class__')
+
+		finally:
+			await browser_session.kill()
+
+	async def test_generate_recent_events_summary_format(self, httpserver: HTTPServer):
+		"""Test that _generate_recent_events_summary produces valid JSON."""
+		browser_session = BrowserSession(browser_profile=BrowserProfile(headless=True, user_data_dir=None, keep_alive=False))
+
+		try:
+			await browser_session.start()
+
+			# Generate some events
+			httpserver.expect_request('/json-test').respond_with_data(
+				'<html><body><h1>JSON Test</h1></body></html>',
+				content_type='text/html',
+			)
+			event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/json-test')))
+			await event
+			await event.event_result(raise_if_any=True, raise_if_none=False)
+
+			# Test the NEW method _generate_recent_events_summary
+			recent_events_json = await browser_session.get_browser_state_summary(include_recent_events=True)
+			recent_events_json = recent_events_json.recent_events
+			assert recent_events_json is not None
+
+			# Should return valid JSON
+			assert recent_events_json != '[]', 'Should have events'
+			events = json.loads(recent_events_json)  # Should not raise JSON decode error
+			assert isinstance(events, list), 'Should return a list of events'
+
+			# Events should exclude problematic fields like 'state'
+			for event in events:
+				assert isinstance(event, dict), 'Each event should be a dict'
+				assert 'state' not in event, "Event should not contain 'state' field (circular reference)"
+
+		finally:
+			await browser_session.kill()
+
+	async def test_event_history_limits(self, httpserver: HTTPServer):
+		"""Test that event history summary respects max_events parameter."""
+		browser_session = BrowserSession(browser_profile=BrowserProfile(headless=True, user_data_dir=None, keep_alive=False))
+
+		try:
+			await browser_session.start()
+
+			# Generate multiple events
+			httpserver.expect_request('/limit-test').respond_with_data(
+				'<html><body><h1>Limit Test</h1></body></html>',
+				content_type='text/html',
+			)
+
+			# Perform multiple actions to generate events
+			event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/limit-test')))
+			await event
+			await event.event_result(raise_if_any=True, raise_if_none=False)
+			screenshot_event = browser_session.event_bus.dispatch(ScreenshotEvent())
+			await screenshot_event
+			await screenshot_event.event_result(raise_if_any=True, raise_if_none=False)
+			await browser_session.get_tabs()
+
+			# Test recent events summary via BrowserStateSummary
+			state_with_events = await browser_session.get_browser_state_summary(include_recent_events=True)
+			assert state_with_events.recent_events is not None
+
+			# Parse the JSON events
+			events = json.loads(state_with_events.recent_events)
+
+			# Should have some events
+			assert len(events) > 0, 'Should have some recent events'
+			assert isinstance(events, list), 'Events should be a list'
+
+		finally:
+			await browser_session.kill()

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_session_start.py

@@ -0,0 +1,434 @@
+"""
+Test script for BrowserSession.start() method to ensure proper initialization,
+concurrency handling, and error handling.
+
+Tests cover:
+- Calling .start() on a session that's already started
+- Simultaneously calling .start() from two parallel coroutines
+- Calling .start() on a session that's started but has a closed browser connection
+- Calling .close() on a session that hasn't been started yet
+"""
+
+import asyncio
+import logging
+
+import pytest
+
+from browser_use.browser.profile import (
+	BROWSERUSE_DEFAULT_CHANNEL,
+	BrowserChannel,
+	BrowserProfile,
+)
+from browser_use.browser.session import BrowserSession
+from browser_use.config import CONFIG
+
+# Set up test logging
+logger = logging.getLogger('browser_session_start_tests')
+# logger.setLevel(logging.DEBUG)
+
+
+# run with pytest -k test_user_data_dir_not_allowed_to_corrupt_default_profile
+
+
+class TestBrowserSessionStart:
+	"""Tests for BrowserSession.start() method initialization and concurrency."""
+
+	@pytest.fixture(scope='module')
+	async def browser_profile(self):
+		"""Create and provide a BrowserProfile with headless mode."""
+		profile = BrowserProfile(headless=True, user_data_dir=None, keep_alive=False)
+		yield profile
+
+	@pytest.fixture(scope='function')
+	async def browser_session(self, browser_profile):
+		"""Create a BrowserSession instance without starting it."""
+		session = BrowserSession(browser_profile=browser_profile)
+		yield session
+		await session.kill()
+
+	async def test_start_already_started_session(self, browser_session):
+		"""Test calling .start() on a session that's already started."""
+		# logger.info('Testing start on already started session')
+
+		# Start the session for the first time
+		await browser_session.start()
+		assert browser_session._cdp_client_root is not None
+
+		# Start the session again - should return immediately without re-initialization
+		await browser_session.start()
+		assert browser_session._cdp_client_root is not None
+
+	# @pytest.mark.skip(reason="Race condition - DOMWatchdog tries to inject scripts into tab that's being closed")
+	# async def test_page_lifecycle_management(self, browser_session: BrowserSession):
+	# 	"""Test session handles page lifecycle correctly."""
+	# 	# logger.info('Testing page lifecycle management')
+
+	# 	# Start the session and get initial state
+	# 	await browser_session.start()
+	# 	initial_tabs = await browser_session.get_tabs()
+	# 	initial_count = len(initial_tabs)
+
+	# 	# Get current tab info
+	# 	current_url = await browser_session.get_current_page_url()
+	# 	assert current_url is not None
+
+	# 	# Get current tab ID
+	# 	current_tab_id = browser_session.agent_focus.target_id if browser_session.agent_focus else None
+	# 	assert current_tab_id is not None
+
+	# 	# Close the current tab using the event system
+	# 	from browser_use.browser.events import CloseTabEvent
+
+	# 	close_event = browser_session.event_bus.dispatch(CloseTabEvent(target_id=current_tab_id))
+	# 	await close_event
+
+	# 	# Operations should still work - may create new page or use existing
+	# 	tabs_after_close = await browser_session.get_tabs()
+	# 	assert isinstance(tabs_after_close, list)
+
+	# 	# Create a new tab explicitly
+	# 	event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url='about:blank', new_tab=True))
+	# 	await event
+	# 	await event.event_result(raise_if_any=True, raise_if_none=False)
+
+	# 	# Should have at least one tab now
+	# 	final_tabs = await browser_session.get_tabs()
+	# 	assert len(final_tabs) >= 1
+
+	async def test_user_data_dir_not_allowed_to_corrupt_default_profile(self):
+		"""Test user_data_dir handling for different browser channels and version mismatches."""
+		# Test 1: Chromium with default user_data_dir and default channel should work fine
+		session = BrowserSession(
+			browser_profile=BrowserProfile(
+				headless=True,
+				user_data_dir=CONFIG.BROWSER_USE_DEFAULT_USER_DATA_DIR,
+				channel=BROWSERUSE_DEFAULT_CHANNEL,  # chromium
+				keep_alive=False,
+			),
+		)
+
+		try:
+			await session.start()
+			assert session._cdp_client_root is not None
+			# Verify the user_data_dir wasn't changed
+			assert session.browser_profile.user_data_dir == CONFIG.BROWSER_USE_DEFAULT_USER_DATA_DIR
+		finally:
+			await session.kill()
+
+		# Test 2: Chrome with default user_data_dir should automatically change dir
+		profile2 = BrowserProfile(
+			headless=True,
+			user_data_dir=CONFIG.BROWSER_USE_DEFAULT_USER_DATA_DIR,
+			channel=BrowserChannel.CHROME,
+			keep_alive=False,
+		)
+
+		# The validator should have changed the user_data_dir to avoid corruption
+		assert profile2.user_data_dir != CONFIG.BROWSER_USE_DEFAULT_USER_DATA_DIR
+		assert profile2.user_data_dir == CONFIG.BROWSER_USE_DEFAULT_USER_DATA_DIR.parent / 'default-chrome'
+
+		# Test 3: Edge with default user_data_dir should also change
+		profile3 = BrowserProfile(
+			headless=True,
+			user_data_dir=CONFIG.BROWSER_USE_DEFAULT_USER_DATA_DIR,
+			channel=BrowserChannel.MSEDGE,
+			keep_alive=False,
+		)
+
+		assert profile3.user_data_dir != CONFIG.BROWSER_USE_DEFAULT_USER_DATA_DIR
+		assert profile3.user_data_dir == CONFIG.BROWSER_USE_DEFAULT_USER_DATA_DIR.parent / 'default-msedge'
+
+
+class TestBrowserSessionReusePatterns:
+	"""Tests for all browser re-use patterns documented in docs/customize/real-browser.mdx"""
+
+	async def test_sequential_agents_same_profile_different_browser(self, mock_llm):
+		"""Test Sequential Agents, Same Profile, Different Browser pattern"""
+		from browser_use import Agent
+		from browser_use.browser.profile import BrowserProfile
+
+		# Create a reusable profile
+		reused_profile = BrowserProfile(
+			user_data_dir=None,  # Use temp dir for testing
+			headless=True,
+		)
+
+		# First agent
+		agent1 = Agent(
+			task='The first task...',
+			llm=mock_llm,
+			browser_profile=reused_profile,
+		)
+		await agent1.run()
+
+		# Verify first agent's session is closed
+		assert agent1.browser_session is not None
+		assert not agent1.browser_session._cdp_client_root is not None
+
+		# Second agent with same profile
+		agent2 = Agent(
+			task='The second task...',
+			llm=mock_llm,
+			browser_profile=reused_profile,
+			# Disable memory for tests
+		)
+		await agent2.run()
+
+		# Verify second agent created a new session
+		assert agent2.browser_session is not None
+		assert agent1.browser_session is not agent2.browser_session
+		assert not agent2.browser_session._cdp_client_root is not None
+
+	async def test_sequential_agents_same_profile_same_browser(self, mock_llm):
+		"""Test Sequential Agents, Same Profile, Same Browser pattern"""
+		from browser_use import Agent, BrowserSession
+
+		# Create a reusable session with keep_alive
+		reused_session = BrowserSession(
+			browser_profile=BrowserProfile(
+				user_data_dir=None,  # Use temp dir for testing
+				headless=True,
+				keep_alive=True,  # Don't close browser after agent.run()
+			),
+		)
+
+		try:
+			# Start the session manually (agents will reuse this initialized session)
+			await reused_session.start()
+
+			# First agent
+			agent1 = Agent(
+				task='The first task...',
+				llm=mock_llm,
+				browser_session=reused_session,
+				# Disable memory for tests
+			)
+			await agent1.run()
+
+			# Verify session is still alive
+			assert reused_session._cdp_client_root is not None
+
+			# Second agent reusing the same session
+			agent2 = Agent(
+				task='The second task...',
+				llm=mock_llm,
+				browser_session=reused_session,
+				# Disable memory for tests
+			)
+			await agent2.run()
+
+			# Verify same browser was used (using __eq__ to check browser_pid, cdp_url)
+			assert agent1.browser_session == agent2.browser_session
+			assert agent1.browser_session == reused_session
+			assert reused_session._cdp_client_root is not None
+
+		finally:
+			await reused_session.kill()
+
+
+class TestBrowserSessionEventSystem:
+	"""Tests for the new event system integration in BrowserSession."""
+
+	@pytest.fixture(scope='function')
+	async def browser_session(self):
+		"""Create a BrowserSession instance for event system testing."""
+		profile = BrowserProfile(headless=True, user_data_dir=None, keep_alive=False)
+		session = BrowserSession(browser_profile=profile)
+		yield session
+		await session.kill()
+
+	async def test_event_bus_initialization(self, browser_session):
+		"""Test that event bus is properly initialized with unique name."""
+		# Event bus should be created during __init__
+		assert browser_session.event_bus is not None
+		assert browser_session.event_bus.name.startswith('EventBus_')
+		# Event bus name format may vary, just check it exists
+
+	async def test_event_handlers_registration(self, browser_session: BrowserSession):
+		"""Test that event handlers are properly registered."""
+		# Attach all watchdogs to register their handlers
+		await browser_session.attach_all_watchdogs()
+
+		# Check that handlers are registered in the event bus
+		from browser_use.browser.events import (
+			BrowserStartEvent,
+			BrowserStateRequestEvent,
+			BrowserStopEvent,
+			ClickElementEvent,
+			CloseTabEvent,
+			ScreenshotEvent,
+			ScrollEvent,
+			TypeTextEvent,
+		)
+
+		# These event types should have handlers registered
+		event_types_with_handlers = [
+			BrowserStartEvent,
+			BrowserStopEvent,
+			ClickElementEvent,
+			TypeTextEvent,
+			ScrollEvent,
+			CloseTabEvent,
+			BrowserStateRequestEvent,
+			ScreenshotEvent,
+		]
+
+		for event_type in event_types_with_handlers:
+			handlers = browser_session.event_bus.handlers.get(event_type.__name__, [])
+			assert len(handlers) > 0, f'No handlers registered for {event_type.__name__}'
+
+	async def test_direct_event_dispatching(self, browser_session):
+		"""Test direct event dispatching without using the public API."""
+		from browser_use.browser.events import BrowserConnectedEvent, BrowserStartEvent
+
+		# Dispatch BrowserStartEvent directly
+		start_event = browser_session.event_bus.dispatch(BrowserStartEvent())
+
+		# Wait for event to complete
+		await start_event
+
+		# Check if BrowserConnectedEvent was dispatched
+		assert browser_session._cdp_client_root is not None
+
+		# Check event history
+		event_history = list(browser_session.event_bus.event_history.values())
+		assert len(event_history) >= 2  # BrowserStartEvent + BrowserConnectedEvent + others
+
+		# Find the BrowserConnectedEvent in history
+		started_events = [e for e in event_history if isinstance(e, BrowserConnectedEvent)]
+		assert len(started_events) >= 1
+		assert started_events[0].cdp_url is not None
+
+	async def test_event_system_error_handling(self, browser_session):
+		"""Test error handling in event system."""
+		from browser_use.browser.events import BrowserStartEvent
+
+		# Create session with invalid CDP URL to trigger error
+		error_session = BrowserSession(
+			browser_profile=BrowserProfile(headless=True),
+			cdp_url='http://localhost:99999',  # Invalid port
+		)
+
+		try:
+			# Dispatch start event directly - should trigger error handling
+			start_event = error_session.event_bus.dispatch(BrowserStartEvent())
+
+			# The event bus catches and logs the error, but the event awaits successfully
+			await start_event
+
+			# The session should not be initialized due to the error
+			assert error_session._cdp_client_root is None, 'Session should not be initialized after connection error'
+
+			# Verify the error was logged in the event history (good enough for error handling test)
+			assert len(error_session.event_bus.event_history) > 0, 'Event should be tracked even with errors'
+
+		finally:
+			await error_session.kill()
+
+	async def test_concurrent_event_dispatching(self, browser_session: BrowserSession):
+		"""Test that concurrent events are handled properly."""
+		from browser_use.browser.events import ScreenshotEvent
+
+		# Start browser first
+		await browser_session.start()
+
+		# Dispatch multiple events concurrently
+		screenshot_event1 = browser_session.event_bus.dispatch(ScreenshotEvent())
+		screenshot_event2 = browser_session.event_bus.dispatch(ScreenshotEvent())
+
+		# Both should complete successfully
+		results = await asyncio.gather(screenshot_event1, screenshot_event2, return_exceptions=True)
+
+		# Check that no exceptions were raised
+		for result in results:
+			assert not isinstance(result, Exception), f'Event failed with: {result}'
+
+	# async def test_many_parallel_browser_sessions(self):
+	# 	"""Test spawning 12 parallel browser_sessions with different settings and ensure they all work"""
+	# 	from browser_use import BrowserSession
+
+	# 	browser_sessions = []
+
+	# 	for i in range(3):
+	# 		browser_sessions.append(
+	# 			BrowserSession(
+	# 				browser_profile=BrowserProfile(
+	# 					user_data_dir=None,
+	# 					headless=True,
+	# 					keep_alive=True,
+	# 				),
+	# 			)
+	# 		)
+	# 	for i in range(3):
+	# 		browser_sessions.append(
+	# 			BrowserSession(
+	# 				browser_profile=BrowserProfile(
+	# 					user_data_dir=Path(tempfile.mkdtemp(prefix=f'browseruse-tmp-{i}')),
+	# 					headless=True,
+	# 					keep_alive=True,
+	# 				),
+	# 			)
+	# 		)
+	# 	for i in range(3):
+	# 		browser_sessions.append(
+	# 			BrowserSession(
+	# 				browser_profile=BrowserProfile(
+	# 					user_data_dir=None,
+	# 					headless=True,
+	# 					keep_alive=False,
+	# 				),
+	# 			)
+	# 		)
+	# 	for i in range(3):
+	# 		browser_sessions.append(
+	# 			BrowserSession(
+	# 				browser_profile=BrowserProfile(
+	# 					user_data_dir=Path(tempfile.mkdtemp(prefix=f'browseruse-tmp-{i}')),
+	# 					headless=True,
+	# 					keep_alive=False,
+	# 				),
+	# 			)
+	# 		)
+
+	# 	print('Starting many parallel browser sessions...')
+	# 	await asyncio.gather(*[browser_session.start() for browser_session in browser_sessions])
+
+	# 	print('Ensuring all parallel browser sessions are connected and usable...')
+	# 	new_tab_tasks = []
+	# 	for browser_session in browser_sessions:
+	# 		assert browser_session._cdp_client_root is not None
+	# 		assert browser_session._cdp_client_root is not None
+	# 		new_tab_tasks.append(browser_session.create_new_tab('chrome://version'))
+	# 	await asyncio.gather(*new_tab_tasks)
+
+	# 	print('killing every 3rd browser_session to test parallel shutdown')
+	# 	kill_tasks = []
+	# 	for i in range(0, len(browser_sessions), 3):
+	# 		kill_tasks.append(browser_sessions[i].kill())
+	# 		browser_sessions[i] = None
+	# 	results = await asyncio.gather(*kill_tasks, return_exceptions=True)
+	# 	# Check that no exceptions were raised during cleanup
+	# 	for i, result in enumerate(results):
+	# 		if isinstance(result, Exception):
+	# 			print(f'Warning: Browser session kill raised exception: {type(result).__name__}: {result}')
+
+	# 	print('ensuring the remaining browser_sessions are still connected and usable')
+	# 	new_tab_tasks = []
+	# 	screenshot_tasks = []
+	# 	for browser_session in filter(bool, browser_sessions):
+	# 		assert browser_session._cdp_client_root is not None
+	# 		assert browser_session._cdp_client_root is not None
+	# 		new_tab_tasks.append(browser_session.create_new_tab('chrome://version'))
+	# 		screenshot_tasks.append(browser_session.take_screenshot())
+	# 	await asyncio.gather(*new_tab_tasks)
+	# 	await asyncio.gather(*screenshot_tasks)
+
+	# 	kill_tasks = []
+	# 	print('killing the remaining browser_sessions')
+	# 	for browser_session in filter(bool, browser_sessions):
+	# 		kill_tasks.append(browser_session.kill())
+	# 	results = await asyncio.gather(*kill_tasks, return_exceptions=True)
+	# 	# Check that no exceptions were raised during cleanup
+	# 	for i, result in enumerate(results):
+	# 		if isinstance(result, Exception):
+	# 			print(f'Warning: Browser session kill raised exception: {type(result).__name__}: {result}')

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_session_tab_management.py

@@ -0,0 +1,316 @@
+import asyncio
+import logging
+
+import pytest
+from dotenv import load_dotenv
+from pytest_httpserver import HTTPServer
+
+load_dotenv()
+
+from browser_use.agent.views import ActionModel
+from browser_use.browser.events import NavigateToUrlEvent
+from browser_use.browser.profile import BrowserProfile
+from browser_use.browser.session import BrowserSession
+from browser_use.tools.service import Tools
+
+# Set up test logging
+logger = logging.getLogger('tab_tests')
+# logger.setLevel(logging.DEBUG)
+
+
+@pytest.fixture(scope='session')
+def http_server():
+	"""Create and provide a test HTTP server that serves static content."""
+	server = HTTPServer()
+	server.start()
+
+	# Add routes for test pages
+	server.expect_request('/page1').respond_with_data(
+		'<html><head><title>Test Page 1</title></head><body><h1>Test Page 1</h1></body></html>', content_type='text/html'
+	)
+	server.expect_request('/page2').respond_with_data(
+		'<html><head><title>Test Page 2</title></head><body><h1>Test Page 2</h1></body></html>', content_type='text/html'
+	)
+	server.expect_request('/page3').respond_with_data(
+		'<html><head><title>Test Page 3</title></head><body><h1>Test Page 3</h1></body></html>', content_type='text/html'
+	)
+	server.expect_request('/page4').respond_with_data(
+		'<html><head><title>Test Page 4</title></head><body><h1>Test Page 4</h1></body></html>', content_type='text/html'
+	)
+
+	yield server
+	server.stop()
+
+
+@pytest.fixture(scope='session')
+def base_url(http_server):
+	"""Return the base URL for the test HTTP server."""
+	return f'http://{http_server.host}:{http_server.port}'
+
+
+@pytest.fixture(scope='module')
+async def browser_session(base_url):
+	"""Create and provide a BrowserSession instance with a properly initialized tab."""
+	browser_session = BrowserSession(
+		browser_profile=BrowserProfile(
+			user_data_dir=None,
+			headless=True,
+			keep_alive=True,
+		)
+	)
+	await browser_session.start()
+
+	# Create an initial tab using the navigate method which is more reliable
+	event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/page1', new_tab=True))
+	await event
+	await event.event_result(raise_if_any=True, raise_if_none=False)
+
+	# Wait for navigation to complete
+	await asyncio.sleep(1)
+
+	# Verify that page is properly set
+	current_url = await browser_session.get_current_page_url()
+	assert base_url in current_url
+
+	# page might be None initially until user interaction occurs
+	# This is expected behavior with the new watchdog architecture
+
+	yield browser_session
+
+	await browser_session.kill()
+
+	# Give playwright time to clean up
+	await asyncio.sleep(0.1)
+
+
+@pytest.fixture(scope='module')
+def tools():
+	"""Create and provide a Tools instance."""
+	return Tools()
+
+
+class TestTabManagement:
+	"""Tests for the tab management system with separate page and page references."""
+
+	# Helper methods
+
+	async def _execute_action(self, tools, browser_session: BrowserSession, action_data):
+		"""Generic helper to execute any action via the tools."""
+		# Dynamically create an appropriate ActionModel class
+		action_type = list(action_data.keys())[0]
+		action_value = action_data[action_type]
+
+		# Create the ActionModel with the single action field
+		class DynamicActionModel(ActionModel):
+			pass
+
+		# Dynamically add the field with the right type annotation
+		setattr(DynamicActionModel, action_type, type(action_value) | None)
+
+		# Execute the action
+		result = await tools.act(DynamicActionModel(**action_data), browser_session)
+
+		# Give the browser a moment to process the action
+		await asyncio.sleep(0.5)
+
+		return result
+
+	async def _reset_tab_state(self, browser_session: BrowserSession, base_url: str):
+		# await browser_session.event_bus.dispatch(CloseTabEvent(target_id=browser_session.agent_focus.target_id))
+		# TODO: close all tabs using events + create new tab + focus it
+		pass
+
+	# Tab management tests
+
+	# async def test_initial_values(self, browser_session, base_url):
+	# 	"""Test that open_tab correctly updates both tab references."""
+
+	# 	await self._reset_tab_state(browser_session, base_url)
+
+	# 	# Get current tab info using the new API
+	# 	current_url = await browser_session.get_current_page_url()
+	# 	assert current_url == 'about:blank'
+	# 	# Note: browser_session.page property may not exist in new architecture
+
+	# 	# Test that get_current_page works even after closing all tabs
+	# 	for page in browser_session._cdp_client_root.pages:
+	# 		await page.close()
+
+	# 	# Give time for watchdogs to process tab closure events
+	# 	await asyncio.sleep(0.5)
+
+	# 	# should never be none even after all pages are closed - new system auto-creates
+	# 	# Check that we can still get current URL (system should auto-create if needed)
+	# 	current_url = await browser_session.get_current_page_url()
+	# 	assert current_url is not None
+	# 	assert current_url == 'about:blank'
+	# run with pytest -k test_agent_changes_tab
+	async def test_agent_changes_tab(self, browser_session: BrowserSession, base_url):
+		"""Test that page changes and page remains the same when a new tab is opened."""
+
+		initial_tab = await self._reset_tab_state(browser_session, base_url)
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/page1'))
+		await event
+		await event.event_result(raise_if_any=True, raise_if_none=False)
+		current_url = await browser_session.get_current_page_url()
+		assert current_url == f'{base_url}/page1'
+		tabs = await browser_session.get_tabs()
+		initial_tab_count = len(tabs)
+
+		# Debug: Check tab count
+		print(f'DEBUG: initial_tab_count = {initial_tab_count}')
+		print(f'DEBUG: browser_session.tabs = {[p.url for p in tabs]}')
+
+		# The test expects 1 tab, but if there's more we need to understand why
+		if initial_tab_count != 1:
+			print(f'WARNING: Expected 1 tab but found {initial_tab_count} tabs after _reset_tab_state')
+			# For now, let's adjust the test to work with the actual count
+			# TODO: fix this initial tab count issue
+			pytest.skip('Initial tab count issue')
+
+		# We expect at least 1 tab but there might be more due to event-driven architecture
+		assert initial_tab_count >= 1
+
+		# test opening a new tab
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/page2', new_tab=True))
+		await event
+		await event.event_result(raise_if_any=True, raise_if_none=False)
+		new_tabs = await browser_session.get_tabs()
+		new_tab_count = len(new_tabs)
+
+		# Debug: Check tab count after new tab creation
+		print(f'DEBUG: new_tab_count = {new_tab_count}')
+		print(f'DEBUG: browser_session.tabs count = {len(new_tabs)}')
+		print(f'DEBUG: browser_session.tabs = {[p.url for p in new_tabs]}')
+
+		# After creating a new tab, we should have one more tab than before
+		expected_new_count = initial_tab_count + 1
+		assert new_tab_count == expected_new_count
+
+		# Give time for watchdogs to process the new tab creation
+		await asyncio.sleep(1.0)
+
+		# test agent open new tab updates agent focus
+		current_url = await browser_session.get_current_page_url()
+		assert current_url == f'{base_url}/page2'
+
+		# test agent navigation updates agent focus
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/page3'))
+		await event
+		await event.event_result(raise_if_any=True, raise_if_none=False)
+		current_url = await browser_session.get_current_page_url()
+		assert current_url == f'{base_url}/page3'  # agent should now be on the new tab
+
+	# async def test_close_tab(self, browser_session, base_url):
+	# 	"""Test that closing a tab updates references correctly."""
+
+	# 	initial_tab = await self._reset_tab_state(browser_session, base_url)
+	# 	event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/page1'))
+	# 	await event
+	# 	await event.event_result(raise_if_any=True, raise_if_none=False)
+	# 	# After navigation, current page should be the correct reference
+	# 	current_url = await browser_session.get_current_page_url()
+	# 	assert current_url == f'{base_url}/page1'
+	# 	# The initial_tab (which was about:blank) should now show the new URL too
+	# 	# (can't check old page object anymore, but current URL confirms navigation worked)
+
+	# 	# Create two tabs with different URLs
+	# 	event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/page2', new_tab=True))
+	# 	await event
+	# 	await event.event_result(raise_if_any=True, raise_if_none=False)
+
+	# 	# Verify the second tab is now active
+	# 	current_url = await browser_session.get_current_page_url()
+	# 	assert current_url == f'{base_url}/page2'
+
+	# 	# Close the second tab using CDP
+	# 	tabs = await browser_session.get_tabs()
+	# 	second_tab_id = None
+	# 	for tab in tabs:
+	# 		if f'{base_url}/page2' in tab.url:
+	# 			second_tab_id = tab.target_id
+	# 			break
+
+	# 	if second_tab_id:
+	# 		event = browser_session.event_bus.dispatch(CloseTabEvent(target_id=second_tab_id))
+	# 		await event
+	# 		await event.event_result(raise_if_any=True, raise_if_none=False)
+	# 	await asyncio.sleep(0.5)
+
+	# 	# Agent reference should be auto-updated to the first available tab
+	# 	current_url = await browser_session.get_current_page_url()
+	# 	assert current_url == f'{base_url}/page1'
+	# 	# (can't check old page object anymore, but current URL confirms tab switch worked)
+
+	# 	# close the only remaining tab using CDP
+	# 	tabs = await browser_session.get_tabs()
+	# 	if tabs:
+	# 		first_tab_id = tabs[0].target_id
+	# 		event = browser_session.event_bus.dispatch(CloseTabEvent(target_id=first_tab_id))
+	# 		await event
+	# 		await event.event_result(raise_if_any=True, raise_if_none=False)
+	# 	await asyncio.sleep(0.5)
+
+	# 	# close_tab should have called get_current_page, which creates a new about:blank tab if none are left
+	# 	current_url = await browser_session.get_current_page_url()
+	# 	assert current_url == 'about:blank'
+
+
+class TestEventDrivenTabOperations:
+	"""Tests for event-driven tab operations introduced in the session refactor."""
+
+	@pytest.fixture(scope='function')
+	async def browser_session(self):
+		"""Create a clean BrowserSession for event testing."""
+		session = BrowserSession(browser_profile=BrowserProfile(headless=True, user_data_dir=None, keep_alive=False))
+		await session.start()
+		yield session
+		await session.kill()
+
+	# async def test_switch_tab_event_dispatching(self, browser_session, base_url):
+	# 	"""Test direct SwitchTabEvent dispatching."""
+
+	# 	# Create multiple tabs
+	# 	await browser_session.navigate_to(f'{base_url}/page1')
+	# 	await browser_session.create_new_tab(f'{base_url}/page2')
+	# 	await browser_session.create_new_tab(f'{base_url}/page3')
+
+	# 	# Switch to tab 0 via direct event
+	# 	switch_event = browser_session.event_bus.dispatch(SwitchTabEvent(target_id=browser_session.tabs[0].target_id))
+	# 	await switch_event
+
+	# 	# Verify the switch worked
+	# 	current_url = await browser_session.get_current_page_url()
+	# 	assert f'{base_url}/page1' in current_url
+
+	# 	# Switch to tab 2 via direct event
+	# 	switch_event = browser_session.event_bus.dispatch(SwitchTabEvent(target_id=browser_session.tabs[2].target_id))
+	# 	await switch_event
+
+	# 	# Verify the switch worked
+	# 	current_url = await browser_session.get_current_page_url()
+	# 	assert f'{base_url}/page3' in current_url
+
+	# async def test_close_tab_event_dispatching(self, browser_session, base_url):
+	# 	"""Test direct CloseTabEvent dispatching."""
+	# 	from browser_use.browser.events import TabClosedEvent
+
+	# 	# Create multiple tabs
+	# 	await browser_session.navigate_to(f'{base_url}/page1')
+	# 	await browser_session.create_new_tab(f'{base_url}/page2')
+
+	# 	initial_tab_count = len(browser_session.tabs)
+	# 	assert initial_tab_count == 2
+
+	# 	# Close tab 1 via direct event
+	# 	close_event = browser_session.event_bus.dispatch(CloseTabEvent(target_id=browser_session.tabs[1].target_id))
+	# 	await close_event
+
+	# 	# Verify tab was closed
+	# 	assert len(browser_session.tabs) == initial_tab_count - 1
+
+	# 	# Check event history for TabClosedEvent
+	# 	event_history = list(browser_session.event_bus.event_history.values())
+	# 	closed_events = [e for e in event_history if isinstance(e, TabClosedEvent)]
+	# 	assert len(closed_events) >= 1
+	# 	assert closed_events[-1].target_id == browser_session.tabs[1].target_id

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_session_take_screenshot.py

@@ -0,0 +1,33 @@
+import pytest
+
+from browser_use.browser.events import NavigateToUrlEvent
+from browser_use.browser.profile import BrowserProfile
+from browser_use.browser.session import BrowserSession
+
+
+@pytest.fixture(scope='function')
+async def browser_session():
+	session = BrowserSession(browser_profile=BrowserProfile(headless=True))
+	await session.start()
+	yield session
+	await session.kill()
+
+
+@pytest.mark.asyncio
+async def test_basic_screenshots(browser_session: BrowserSession, httpserver):
+	"""Navigate to a local page and ensure screenshot helpers return bytes."""
+
+	html = """
+    <html><body><h1 id='title'>Hello</h1><p>Screenshot demo.</p></body></html>
+    """
+	httpserver.expect_request('/demo').respond_with_data(html, content_type='text/html')
+	url = httpserver.url_for('/demo')
+
+	nav = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=url, new_tab=False))
+	await nav
+
+	data = await browser_session.take_screenshot(full_page=False)
+	assert data, 'Viewport screenshot returned no data'
+
+	element = await browser_session.screenshot_element('h1')
+	assert element, 'Element screenshot returned no data'

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_session_via_cdp_tab_management.py

@@ -0,0 +1,121 @@
+"""Test CDP session handling when creating new tabs."""
+
+import asyncio
+
+import pytest
+
+from browser_use.browser.events import NavigateToUrlEvent, TabCreatedEvent
+from browser_use.browser.profile import BrowserProfile, ViewportSize
+from browser_use.browser.session import BrowserSession
+
+
+@pytest.fixture
+async def httpserver_url(httpserver):
+	"""Create a local HTTP server for testing."""
+	httpserver.expect_request('/').respond_with_data(
+		"""
+		<html>
+		<head><title>Test Page</title></head>
+		<body>
+			<h1>Test Page</h1>
+			<p>This is a test page</p>
+		</body>
+		</html>
+		""",
+		content_type='text/html',
+	)
+	return httpserver.url_for('/')
+
+
+@pytest.mark.skip(reason='TODO: fix')
+async def test_new_tab_cdp_session_attachment(httpserver_url):
+	"""Test that CDP session is properly attached when creating new tabs."""
+	browser = BrowserSession(browser_profile=BrowserProfile(headless=True, viewport=ViewportSize(width=800, height=600)))
+
+	tab_created_events = []
+
+	# Track TabCreatedEvent to verify it's dispatched correctly
+	browser.event_bus.on(TabCreatedEvent, lambda event: tab_created_events.append(event))
+
+	try:
+		await browser.start()
+
+		# Navigate to initial page
+		nav_event = browser.event_bus.dispatch(NavigateToUrlEvent(url=httpserver_url))
+		await nav_event
+
+		# Clear any initial tab created events
+		tab_created_events.clear()
+
+		# Now create a new tab - this should trigger the CDP error if not fixed
+		new_tab_event = browser.event_bus.dispatch(NavigateToUrlEvent(url=httpserver_url, new_tab=True))
+		await new_tab_event
+
+		# Wait a bit for all events to process
+		await asyncio.sleep(1)
+
+		# Verify that TabCreatedEvent was dispatched
+		assert len(tab_created_events) == 1, f'Expected 1 TabCreatedEvent, got {len(tab_created_events)}'
+		assert tab_created_events[0].url == httpserver_url
+
+		# Verify we have 2 tabs now
+		tabs = await browser.get_tabs()
+		assert len(tabs) == 2, f'Expected 2 tabs, got {len(tabs)}'
+
+		# Verify the CDP session is attached to the new tab
+		assert browser.agent_focus is not None
+		assert browser.agent_focus.target_id is not None
+
+		# Try to execute a CDP command on the new tab to verify it works
+		cdp_session = await browser.get_or_create_cdp_session()
+		result = await cdp_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': 'document.title'}, session_id=cdp_session.session_id
+		)
+		assert result['result'].get('value') == 'Test Page'
+
+		# Get browser state to verify DOM can be built on new tab
+		from browser_use.browser.events import BrowserStateRequestEvent
+
+		state_event = browser.event_bus.dispatch(BrowserStateRequestEvent())
+		state = await state_event.event_result()
+
+		# Verify state was retrieved without errors
+		assert state is not None
+		assert state.dom_state is not None
+
+	finally:
+		await browser.stop()
+
+
+async def test_multiple_new_tabs_cdp_session(httpserver_url):
+	"""Test creating multiple new tabs in succession."""
+	browser = BrowserSession(browser_profile=BrowserProfile(headless=True, viewport=ViewportSize(width=800, height=600)))
+
+	try:
+		await browser.start()
+
+		# Navigate to initial page
+		nav_event = browser.event_bus.dispatch(NavigateToUrlEvent(url=httpserver_url))
+		await nav_event
+
+		# Create multiple new tabs quickly
+		for i in range(3):
+			new_tab_event = browser.event_bus.dispatch(NavigateToUrlEvent(url=f'{httpserver_url}?tab={i}', new_tab=True))
+			await new_tab_event
+
+		# Wait for events to process
+		await asyncio.sleep(1)
+
+		# Verify we have 4 tabs total (1 initial + 3 new)
+		tabs = await browser.get_tabs()
+		assert len(tabs) == 4, f'Expected 4 tabs, got {len(tabs)}'
+
+		# Verify CDP commands work on the current tab
+		cdp_session = await browser.get_or_create_cdp_session()
+		result = await cdp_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': 'window.location.href'}, session_id=cdp_session.session_id
+		)
+		assert 'tab=2' in result['result'].get('value', ''), 'Should be on the last created tab'
+
+	finally:
+		await browser.stop()

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_session_viewport_and_proxy.py

@@ -0,0 +1,21 @@
+async def test_proxy_settings_pydantic_model():
+	"""
+	Test that ProxySettings as a Pydantic model is correctly converted to a dictionary when used.
+	"""
+	# Create ProxySettings with Pydantic model
+	proxy_settings = dict(server='http://example.proxy:4242', bypass='localhost', username='testuser', password='testpass')
+
+	# Verify the model has correct dict-like access
+	assert proxy_settings['server'] == 'http://example.proxy:4242'
+	assert proxy_settings.get('bypass') == 'localhost'
+	assert proxy_settings.get('nonexistent', 'default') == 'default'
+
+	# Verify model_dump works correctly
+	proxy_dict = dict(proxy_settings)
+	assert isinstance(proxy_dict, dict)
+	assert proxy_dict['server'] == 'http://example.proxy:4242'
+	assert proxy_dict['bypass'] == 'localhost'
+	assert proxy_dict['username'] == 'testuser'
+	assert proxy_dict['password'] == 'testpass'
+
+	# We don't launch the actual browser - we just verify the model itself works as expected

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_watchdog_crash.py

@@ -0,0 +1,328 @@
+"""Test CrashWatchdog functionality."""
+
+import asyncio
+from typing import cast
+
+import pytest
+
+from browser_use.browser.events import (
+	BrowserConnectedEvent,
+	BrowserErrorEvent,
+	BrowserStartEvent,
+	BrowserStopEvent,
+	BrowserStoppedEvent,
+	NavigateToUrlEvent,
+)
+from browser_use.browser.profile import BrowserProfile
+from browser_use.browser.session import BrowserSession
+from browser_use.utils import logger
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip('CrashWatchdog not implemented in current CDP architecture')
+async def test_crash_watchdog_network_timeout():
+	"""Test that CrashWatchdog detects network timeouts by monitoring actual network requests."""
+
+	# Create browser session
+	profile = BrowserProfile(headless=True)
+	session = BrowserSession(browser_profile=profile)
+
+	try:
+		# Start browser using event system
+		session.event_bus.dispatch(BrowserStartEvent())
+		await session.event_bus.expect(BrowserConnectedEvent, timeout=10.0)
+
+		logger.info('[TEST] Browser started, configuring watchdog timeout')
+
+		# Configure crash watchdog with very short timeout for testing
+		if hasattr(session, '_crash_watchdog') and session._crash_watchdog:
+			session._crash_watchdog.network_timeout_seconds = 1.0  # Very short timeout
+			session._crash_watchdog.check_interval_seconds = 0.2  # Check frequently
+
+		# Try to navigate to a non-existent slow server that will hang
+		# This will create a real network request that will timeout
+		slow_url = 'http://192.0.2.1:4242/timeout-test'  # RFC5737 TEST-NET-1 - non-routable
+		logger.info(f'[TEST] Navigating to non-routable URL via events: {slow_url}')
+		session.event_bus.dispatch(NavigateToUrlEvent(url=slow_url))
+
+		# Wait for the network timeout error via event bus
+		logger.info('[TEST] Waiting for NetworkTimeout event via event bus...')
+		try:
+			timeout_error = cast(
+				BrowserErrorEvent,
+				await session.event_bus.expect(
+					BrowserErrorEvent, predicate=lambda e: cast(BrowserErrorEvent, e).error_type == 'NetworkTimeout', timeout=8.0
+				),
+			)
+
+			# Verify the timeout event details
+			assert 'timeout-test' in timeout_error.details['url']
+			assert timeout_error.details['elapsed_seconds'] >= 0.8  # Should be at least close to our timeout
+			assert timeout_error.message.startswith('Network request timed out after')
+
+			logger.info(f'[TEST] Successfully detected network timeout: {timeout_error.message}')
+		except TimeoutError:
+			# Network timeout detection can be flaky in test environment
+			logger.warning('[TEST] NetworkTimeout event not received - this is expected in some test environments')
+
+			# Verify the crash watchdog is running and configured correctly
+			assert session._crash_watchdog is not None, 'CrashWatchdog should exist'
+			assert session._crash_watchdog.network_timeout_seconds == 1.0, 'Network timeout should be configured'
+			assert session._crash_watchdog._monitoring_task is not None, 'Monitoring task should be running'
+			assert not session._crash_watchdog._monitoring_task.done(), 'Monitoring task should still be active'
+
+			logger.info('[TEST] Crash watchdog is properly configured and running - test passes')
+
+	finally:
+		# Clean shutdown
+		try:
+			session.event_bus.dispatch(BrowserStopEvent())
+			await session.event_bus.expect(BrowserStoppedEvent, timeout=3.0)
+		except Exception:
+			# If graceful shutdown fails, force cleanup
+			await session.kill()
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip('CrashWatchdog not implemented in current CDP architecture')
+async def test_crash_watchdog_browser_disconnect():
+	"""Test that CrashWatchdog detects browser disconnection through monitoring."""
+	profile = BrowserProfile(headless=True)
+	session = BrowserSession(browser_profile=profile)
+
+	try:
+		# Start browser
+		start_event = session.event_bus.dispatch(BrowserStartEvent())
+		await start_event
+		# Ensure any exceptions from the event handler are propagated
+		await start_event.event_result(raise_if_any=True, raise_if_none=False)
+
+		# Wait for browser to be fully started
+		await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
+
+		# Browser disconnection detection is now handled by the crash watchdog
+		# No configuration needed
+
+		# Mock browser disconnection by overriding is_connected
+		# This simulates what would happen if the browser process crashed
+		if session._cdp_client_root:
+			# Simulate disconnection by setting _cdp_client_root to None
+			original_cdp_client = session._cdp_client_root
+			session._cdp_client_root = None
+
+			try:
+				# Wait for watchdog to detect disconnection
+				disconnect_error: BrowserErrorEvent = cast(
+					BrowserErrorEvent,
+					await session.event_bus.expect(
+						BrowserErrorEvent,
+						predicate=lambda e: cast(BrowserErrorEvent, e).error_type == 'BrowserDisconnected',
+						timeout=2.0,
+					),
+				)
+				assert 'disconnected unexpectedly' in disconnect_error.message
+			finally:
+				# Restore original CDP client
+				session._cdp_client_root = original_cdp_client
+
+	finally:
+		# Force stop even if browser is marked as disconnected
+		try:
+			session.event_bus.dispatch(BrowserStopEvent(force=True))
+			await asyncio.sleep(0.5)  # Give it time to stop
+		except Exception:
+			pass  # Browser might already be stopped
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip('CrashWatchdog not implemented in current CDP architecture')
+async def test_crash_watchdog_lifecycle():
+	"""Test that CrashWatchdog starts and stops with browser session."""
+	profile = BrowserProfile(headless=True)
+	session = BrowserSession(browser_profile=profile)
+
+	# Start browser via event and wait for BrowserConnectedEvent
+	start_event = session.event_bus.dispatch(BrowserStartEvent())
+	await start_event  # Wait for the event and all handlers to complete
+
+	started_event: BrowserConnectedEvent = cast(
+		BrowserConnectedEvent, await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
+	)
+	assert started_event.cdp_url is not None
+
+	# Verify crash watchdog is running
+	assert hasattr(session, '_crash_watchdog'), 'CrashWatchdog should be created'
+	assert session._crash_watchdog is not None, 'CrashWatchdog should not be None'
+
+	# Check monitoring task is active
+	assert session._crash_watchdog._monitoring_task is not None
+	assert not session._crash_watchdog._monitoring_task.done()
+
+	# Stop browser via event
+	session.event_bus.dispatch(BrowserStopEvent())
+
+	# Wait for browser stopped event
+	try:
+		stopped_event: BrowserStoppedEvent = cast(
+			BrowserStoppedEvent, await session.event_bus.expect(BrowserStoppedEvent, timeout=3.0)
+		)
+		assert stopped_event.reason is not None
+	except TimeoutError:
+		# Browser stop can be flaky in test environment
+		logger.warning('[TEST] BrowserStoppedEvent timeout - this is expected in some test environments')
+		# Just verify the crash watchdog exists
+		assert session._crash_watchdog is not None
+
+	# Verify monitoring task was stopped
+	await asyncio.sleep(0.1)  # Give it a moment to clean up
+	if session._crash_watchdog._monitoring_task:
+		assert session._crash_watchdog._monitoring_task.done()
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip(reason='Browser initialization timeout in test environment - timing issue')
+async def test_infinite_loop_page_blocking():
+	"""Test that pages with infinite JavaScript loops are detected as unresponsive."""
+	from pytest_httpserver import HTTPServer
+
+	# Create HTTP server with blocking page
+	httpserver = HTTPServer()
+	httpserver.start()
+
+	# Add route that serves permanently blocking JavaScript
+	httpserver.expect_request('/infinite-loop').respond_with_data(
+		'<html><body><h1>Loading...</h1><script>while(true){}</script></body></html>', content_type='text/html'
+	)
+
+	profile = BrowserProfile(headless=True)
+	session = BrowserSession(browser_profile=profile)
+
+	try:
+		# Start browser
+		session.event_bus.dispatch(BrowserStartEvent())
+		await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
+
+		# Navigate to blocking page
+		blocking_url = httpserver.url_for('/infinite-loop')
+		session.event_bus.dispatch(NavigateToUrlEvent(url=blocking_url))
+
+		# The navigation should timeout or trigger an error
+		# We don't expect NavigationCompleteEvent since the page blocks
+		await asyncio.sleep(2)  # Give it time to detect the issue
+
+		# Try to interact with the page via CDP - should still work at protocol level
+		cdp_session = await session.get_or_create_cdp_session()
+
+		# CDP commands should still work even if page is blocked
+		version_result = await session.cdp_client.send.Browser.getVersion()
+		assert version_result is not None
+
+		# Close the blocking tab to recover
+		await session.cdp_client.send.Target.closeTarget(params={'targetId': cdp_session.target_id})
+
+	finally:
+		httpserver.stop()
+		session.event_bus.dispatch(BrowserStopEvent())
+		await asyncio.sleep(0.5)
+
+
+# @pytest.mark.asyncio
+# async def test_transient_blocking_recovery():
+# 	"""Test recovery from temporarily blocking JavaScript."""
+# 	from pytest_httpserver import HTTPServer
+
+# 	httpserver = HTTPServer()
+# 	httpserver.start()
+
+# 	# Page that blocks for 1 second then recovers
+# 	httpserver.expect_request('/transient-block').respond_with_data(
+# 		"""<html><body>
+# 		<h1 id="status">Blocking...</h1>
+# 		<script>
+# 			const start = Date.now();
+# 			while (Date.now() - start < 1000) {} // Block for 1 second
+# 			document.getElementById('status').textContent = 'Recovered!';
+# 		</script>
+# 		</body></html>""",
+# 		content_type='text/html',
+# 	)
+
+# 	profile = BrowserProfile(headless=True)
+# 	session = BrowserSession(browser_profile=profile)
+
+# 	try:
+# 		# Start browser
+# 		session.event_bus.dispatch(BrowserStartEvent())
+# 		await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
+
+# 		# Navigate to transiently blocking page
+# 		url = httpserver.url_for('/transient-block')
+# 		session.event_bus.dispatch(NavigateToUrlEvent(url=url))
+
+# 		# Wait for the blocking to end
+# 		await asyncio.sleep(2)
+
+# 		# Verify page recovered and we can interact with it
+# 		cdp_session = await session.get_or_create_cdp_session()
+# 		result = await session.cdp_client.send.Runtime.evaluate(
+# 			params={'expression': 'document.getElementById("status").textContent', 'returnByValue': True},
+# 			session_id=cdp_session.session_id,
+# 		)
+
+# 		status_text = result.get('result', {}).get('value', '')
+# 		assert status_text == 'Recovered!', f"Expected 'Recovered!' but got '{status_text}'"
+
+# 	finally:
+# 		httpserver.stop()
+# 		session.event_bus.dispatch(BrowserStopEvent())
+# 		await asyncio.sleep(0.5)
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip(reason='Browser initialization timeout in test environment - timing issue')
+async def test_browser_process_kill_detection():
+	"""Test that killing the browser process is detected."""
+	import os
+	import signal
+
+	profile = BrowserProfile(headless=True)
+	session = BrowserSession(browser_profile=profile)
+
+	try:
+		# Start browser
+		session.event_bus.dispatch(BrowserStartEvent())
+		await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
+
+		# Get browser process PID
+		browser_pid = None
+		if session._local_browser_watchdog and session._local_browser_watchdog._subprocess:
+			browser_pid = session._local_browser_watchdog._subprocess.pid
+
+		if browser_pid:
+			# Kill the browser process
+			try:
+				os.kill(browser_pid, signal.SIGKILL)
+			except ProcessLookupError:
+				pass  # Process might already be gone
+
+			# Wait for crash detection
+			try:
+				error_event = cast(
+					BrowserErrorEvent,
+					await session.event_bus.expect(
+						BrowserErrorEvent,
+						predicate=lambda e: 'disconnect' in cast(BrowserErrorEvent, e).message.lower(),
+						timeout=5.0,
+					),
+				)
+				assert error_event is not None
+			except TimeoutError:
+				# Crash detection might not trigger in all environments
+				pass
+
+	finally:
+		# Force cleanup
+		try:
+			await session.kill()
+		except Exception:
+			pass

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_watchdog_downloads.py

@@ -0,0 +1,265 @@
+"""Test downloads watchdog functionality."""
+
+import asyncio
+import tempfile
+from pathlib import Path
+
+import pytest
+from pytest_httpserver import HTTPServer
+
+from browser_use.browser import BrowserSession
+from browser_use.browser.events import (
+	BrowserConnectedEvent,
+	BrowserStartEvent,
+	BrowserStopEvent,
+	BrowserStoppedEvent,
+	NavigateToUrlEvent,
+)
+from browser_use.browser.profile import BrowserProfile
+
+
+@pytest.mark.skip(reason='TODO: fix')
+async def test_downloads_watchdog_lifecycle():
+	"""Test that DownloadsWatchdog starts and stops with browser session."""
+
+	# Create temp directory for downloads
+	with tempfile.TemporaryDirectory() as temp_dir:
+		downloads_path = Path(temp_dir)
+
+		profile = BrowserProfile(headless=True, downloads_path=downloads_path)
+		session = BrowserSession(browser_profile=profile)
+
+		# Check that downloads watchdog is None initially
+		assert session._downloads_watchdog is None
+
+		try:
+			# Start browser
+			session.event_bus.dispatch(BrowserStartEvent())
+			await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
+
+			# Check that downloads watchdog is attached
+			assert session._downloads_watchdog is not None
+
+			# Verify watchdog has proper session reference
+			assert session._downloads_watchdog.browser_session is session
+
+		finally:
+			# Clean shutdown
+			try:
+				session.event_bus.dispatch(BrowserStopEvent())
+				await session.event_bus.expect(BrowserStoppedEvent, timeout=3.0)
+			except Exception:
+				# If graceful shutdown fails, force cleanup
+				await session.kill()
+			# Always stop event bus to prevent hanging
+			await session.event_bus.stop(clear=True, timeout=5)
+
+
+@pytest.mark.skip(reason='TODO: fix')
+async def test_downloads_watchdog_file_detection(download_test_server):
+	"""Test that DownloadsWatchdog detects file downloads."""
+
+	# Create temp directory for downloads
+	with tempfile.TemporaryDirectory() as temp_dir:
+		downloads_path = Path(temp_dir)
+
+		profile = BrowserProfile(headless=True, downloads_path=downloads_path)
+		session = BrowserSession(browser_profile=profile)
+
+		try:
+			# Start browser
+			session.event_bus.dispatch(BrowserStartEvent())
+			await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
+
+			# Navigate to test page
+			test_url = download_test_server.url_for('/')
+			await session.event_bus.dispatch(NavigateToUrlEvent(url=test_url))
+
+			# Skip complex element selection for now - would need to implement selector-to-index conversion
+			pytest.skip('Complex element selection needs refactoring for CDP events')
+
+			# Wait for download to complete
+			await asyncio.sleep(2.0)
+
+			# Verify file was downloaded
+			downloaded_files = list(downloads_path.glob('*'))
+			assert len(downloaded_files) == 1, f'Expected 1 file, got {len(downloaded_files)}: {downloaded_files}'
+
+			# Verify file content
+			downloaded_file = downloaded_files[0]
+			assert downloaded_file.name == 'test.pdf'
+			assert downloaded_file.read_bytes() == b'PDF content'
+
+		finally:
+			# Clean shutdown
+			try:
+				session.event_bus.dispatch(BrowserStopEvent())
+				await session.event_bus.expect(BrowserStoppedEvent, timeout=3.0)
+			except Exception:
+				# If graceful shutdown fails, force cleanup
+				await session.kill()
+			# Always stop event bus to prevent hanging
+			await session.event_bus.stop(clear=True, timeout=5)
+
+
+@pytest.fixture
+def comprehensive_download_test_server():
+	"""Create a test server with downloadable files."""
+	httpserver = HTTPServer(host='127.0.0.1', port=0)
+	httpserver.start()
+
+	# Serve a main page with download links
+	main_page_html = """
+	<!DOCTYPE html>
+	<html>
+	<head>
+		<title>Download Test Page</title>
+	</head>
+	<body>
+		<h1>Download Test Page</h1>
+		<a href="/download/test.pdf" download="test.pdf">Download PDF</a>
+		<br>
+		<a href="/download/test.txt" download="test.txt">Download Text</a>
+	</body>
+	</html>
+	"""
+
+	httpserver.expect_request('/').respond_with_data(main_page_html, content_type='text/html')
+
+	# PDF handler
+	httpserver.expect_request('/download/test.pdf').respond_with_data(
+		b'PDF content', status=200, headers={'Content-Type': 'application/pdf'}
+	)
+
+	# Text handler
+	httpserver.expect_request('/download/test.txt').respond_with_data(
+		b'Text content', status=200, headers={'Content-Type': 'text/plain'}
+	)
+
+	yield httpserver
+	httpserver.stop()
+
+
+# @pytest.mark.asyncio
+# async def test_downloads_watchdog_page_attachment():
+# 	"""Test that DownloadsWatchdog attaches to pages properly."""
+
+# 	# Create temp directory for downloads
+# 	with tempfile.TemporaryDirectory() as temp_dir:
+# 		downloads_path = Path(temp_dir)
+
+# 		profile = BrowserProfile(headless=True, downloads_path=downloads_path)
+# 		session = BrowserSession(browser_profile=profile)
+
+# 		try:
+# 			# Start browser
+# 			session.event_bus.dispatch(BrowserStartEvent())
+# 			await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
+
+# 			# Get downloads watchdog
+# 			downloads_watchdog = session._downloads_watchdog
+# 			assert downloads_watchdog is not None
+
+# 			# Navigate to create a new page
+# 			event = session.event_bus.dispatch(NavigateToUrlEvent(url='data:text/html,<h1>Test Page</h1>'))
+# 			await event
+# 			await event.event_result(raise_if_any=True, raise_if_none=False)
+
+# 			# Verify watchdog has pages with listeners
+# 			assert hasattr(downloads_watchdog, '_pages_with_listeners')
+
+# 			# Give it a moment for page attachment
+# 			await asyncio.sleep(0.2)
+
+# 			# The watchdog should have attached to at least one page
+# 			# Note: We can't easily verify the internal WeakSet without accessing private attrs
+
+# 		finally:
+# 			# Clean shutdown
+# 			try:
+# 				session.event_bus.dispatch(BrowserStopEvent())
+# 				await session.event_bus.expect(BrowserStoppedEvent, timeout=3.0)
+# 			except Exception:
+# 				# If graceful shutdown fails, force cleanup
+# 				await session.kill()
+# 			# Always stop event bus to prevent hanging
+# 			await session.event_bus.stop(clear=True, timeout=5)
+
+
+# @pytest.mark.asyncio
+# async def test_downloads_watchdog_default_downloads_path():
+# 	"""Test that DownloadsWatchdog works with default downloads path."""
+
+# 	# Don't specify downloads path - should use default
+# 	profile = BrowserProfile(headless=True)
+# 	session = BrowserSession(browser_profile=profile)
+
+# 	try:
+# 		# Start browser
+# 		session.event_bus.dispatch(BrowserStartEvent())
+# 		await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
+
+# 		# Verify downloads watchdog is attached
+# 		assert session._downloads_watchdog is not None
+
+# 		# The default downloads path should be set
+# 		# Note: We can't easily test the actual download without complex setup
+
+# 	finally:
+# 		# Clean shutdown
+# 		try:
+# 			session.event_bus.dispatch(BrowserStopEvent())
+# 			await session.event_bus.expect(BrowserStoppedEvent, timeout=3.0)
+# 		except Exception:
+# 			# If graceful shutdown fails, force cleanup
+# 			await session.kill()
+# 		# Always stop event bus to prevent hanging
+# 		await session.event_bus.stop(clear=True, timeout=5)
+
+
+# @pytest.mark.asyncio
+# async def test_unique_downloads_directories():
+# 	"""Test that different browser profiles get unique downloads directories."""
+
+# 	# Create temp directory for downloads
+# 	with tempfile.TemporaryDirectory() as temp_dir:
+# 		downloads_path_1 = Path(temp_dir) / 'downloads1'
+# 		downloads_path_2 = Path(temp_dir) / 'downloads2'
+# 		downloads_path_1.mkdir()
+# 		downloads_path_2.mkdir()
+
+# 		profile1 = BrowserProfile(headless=True, downloads_path=downloads_path_1)
+# 		profile2 = BrowserProfile(headless=True, downloads_path=downloads_path_2)
+
+# 		session1 = BrowserSession(browser_profile=profile1)
+# 		session2 = BrowserSession(browser_profile=profile2)
+
+# 		try:
+# 			# Start both browsers
+# 			session1.event_bus.dispatch(BrowserStartEvent())
+# 			await session1.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
+
+# 			session2.event_bus.dispatch(BrowserStartEvent())
+# 			await session2.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
+
+# 			# Verify downloads watchdogs are attached
+# 			assert session1._downloads_watchdog is not None
+# 			assert session2._downloads_watchdog is not None
+
+# 			# Verify they have different downloads paths
+# 			assert session1.browser_profile.downloads_path != session2.browser_profile.downloads_path
+
+# 		finally:
+# 			# Clean shutdown both sessions
+# 			for session in [session1, session2]:
+# 				try:
+# 					session.event_bus.dispatch(BrowserStopEvent())
+# 					await session.event_bus.expect(BrowserStoppedEvent, timeout=3.0)
+# 				except Exception:
+# 					# If graceful shutdown fails, force cleanup
+# 					await session.kill()
+# 				# Always stop event bus to prevent hanging
+# 				await session.event_bus.stop(clear=True, timeout=5)
+
+
+# Removed test_downloads_watchdog_actual_download_detection - complex Playwright patterns not suitable for CDP

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_watchdog_downloads_simple.py

@@ -0,0 +1,33 @@
+"""Test simple download functionality."""
+
+import pytest
+
+# Skip Playwright imports - removed dependency
+from pytest_httpserver import HTTPServer
+
+
+async def test_simple_playwright_download():
+	"""Test basic Playwright download functionality without browser-use - this just validates the browser setup"""
+	# Skip Playwright usage - removed dependency
+	pytest.skip('Playwright dependency removed')
+
+
+@pytest.fixture(scope='function')
+def http_server():
+	"""Create a test HTTP server with a downloadable file."""
+	server = HTTPServer()
+	server.start()
+
+	# Serve a simple text file for download
+	server.expect_request('/download/test.txt').respond_with_data(
+		'Hello World from HTTP Server', status=200, headers={'Content-Type': 'text/plain'}
+	)
+
+	yield server
+	server.stop()
+
+
+async def test_browser_use_download_with_http_server(http_server):
+	"""Test browser-use download with HTTP server and event coordination"""
+	# Skip complex element selection for now - would need to implement selector-to-index conversion
+	pytest.skip('Complex element selection needs refactoring for CDP events')

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_watchdog_downloads_upload_full_circle.py

@@ -0,0 +1,367 @@
+"""Test full circle: download a file and then upload it back, verifying hash matches"""
+
+import asyncio
+import hashlib
+import tempfile
+from pathlib import Path
+
+import pytest
+from pytest_httpserver import HTTPServer
+
+from browser_use.agent.views import ActionModel
+from browser_use.browser import BrowserSession
+from browser_use.browser.events import BrowserStateRequestEvent, FileDownloadedEvent
+from browser_use.browser.profile import BrowserProfile
+from browser_use.filesystem.file_system import FileSystem
+from browser_use.tools.service import Tools
+from browser_use.tools.views import ClickElementAction, GoToUrlAction, UploadFileAction
+
+
+@pytest.fixture(scope='function')
+def download_upload_server():
+	"""Create a test HTTP server with download and upload endpoints."""
+	server = HTTPServer()
+	server.start()
+
+	# Test file content and hash
+	test_content = b'This is a test file for download-upload verification. Random: 12345'
+	test_hash = hashlib.sha256(test_content).hexdigest()
+
+	# Store uploaded files data
+	uploaded_files = []
+
+	# Add download endpoint
+	server.expect_request('/download/test-file.txt').respond_with_data(
+		test_content, content_type='text/plain', headers={'Content-Disposition': 'attachment; filename="test-file.txt"'}
+	)
+
+	# Add upload page
+	upload_page_html = """
+	<!DOCTYPE html>
+	<html>
+	<head>
+		<title>Upload Test Page</title>
+	</head>
+	<body>
+		<h1>File Upload Test</h1>
+		<form id="uploadForm" action="/upload" method="POST" enctype="multipart/form-data">
+			<input type="file" id="fileInput" name="file" />
+			<button type="submit" id="submitButton">Upload File</button>
+		</form>
+		<div id="result"></div>
+		
+		<script>
+			document.getElementById('uploadForm').addEventListener('submit', async (e) => {
+				e.preventDefault();
+				const formData = new FormData(e.target);
+				const file = formData.get('file');
+				
+				if (file) {
+					// Read file content
+					const content = await file.text();
+					
+					// Calculate SHA256 hash
+					const encoder = new TextEncoder();
+					const data = encoder.encode(content);
+					const hashBuffer = await crypto.subtle.digest('SHA-256', data);
+					const hashArray = Array.from(new Uint8Array(hashBuffer));
+					const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
+					
+					// Display result
+					document.getElementById('result').innerHTML = `
+						<p>File uploaded successfully!</p>
+						<p>Filename: <span id="uploadedFileName">${file.name}</span></p>
+						<p>Size: <span id="uploadedFileSize">${file.size}</span> bytes</p>
+						<p>SHA256: <span id="uploadedFileHash">${hashHex}</span></p>
+					`;
+					
+					// Send to server for verification
+					fetch('/upload', {
+						method: 'POST',
+						body: formData
+					});
+				}
+			});
+		</script>
+	</body>
+	</html>
+	"""
+
+	server.expect_request('/upload-page').respond_with_data(upload_page_html, content_type='text/html')
+
+	# Handle upload POST request (for server-side verification)
+	def handle_upload(request):
+		# Store uploaded file info for verification
+		if request.files and 'file' in request.files:
+			file_data = request.files['file'][0]
+			uploaded_files.append(
+				{
+					'filename': file_data['filename'],
+					'content': file_data['body'],
+					'hash': hashlib.sha256(file_data['body']).hexdigest(),
+				}
+			)
+		return request.make_response({'status': 'ok'})
+
+	server.expect_request('/upload', method='POST').respond_with_handler(handle_upload)
+
+	# Add download page with link
+	download_page_html = f"""
+	<!DOCTYPE html>
+	<html>
+	<head>
+		<title>Download Test Page</title>
+	</head>
+	<body>
+		<h1>File Download Test</h1>
+		<a id="downloadLink" href="/download/test-file.txt">Download Test File</a>
+		<p>Original file SHA256: <span id="originalHash">{test_hash}</span></p>
+	</body>
+	</html>
+	"""
+
+	server.expect_request('/download-page').respond_with_data(download_page_html, content_type='text/html')
+
+	# stop
+	server.stop()
+	# Skip complex HTTPServer attribute assignment - not supported
+	pytest.skip('Complex HTTPServer attribute assignment not supported')
+
+	yield server
+	server.stop()
+
+
+class TestDownloadUploadFullCircle:
+	"""Test full circle: download a file and then upload it back"""
+
+	async def test_download_then_upload_with_hash_verification(self, download_upload_server):
+		"""Download a file, then upload it to another page, verify hash matches"""
+
+		# Create temporary directory for downloads
+		with tempfile.TemporaryDirectory() as tmpdir:
+			downloads_path = Path(tmpdir) / 'downloads'
+			downloads_path.mkdir()
+
+			# Create browser session with downloads enabled
+			browser_session = BrowserSession(
+				browser_profile=BrowserProfile(
+					headless=True,
+					downloads_path=str(downloads_path),
+					user_data_dir=None,
+				)
+			)
+
+			await browser_session.start()
+
+			# Create tools and file system
+			tools = Tools()
+			file_system = FileSystem(base_dir=tmpdir)
+
+			try:
+				base_url = f'http://{download_upload_server.host}:{download_upload_server.port}'
+
+				# Step 1: Navigate to download page
+				class GoToUrlActionModel(ActionModel):
+					go_to_url: GoToUrlAction | None = None
+
+				result = await tools.act(
+					GoToUrlActionModel(go_to_url=GoToUrlAction(url=f'{base_url}/download-page', new_tab=False)), browser_session
+				)
+				assert result.error is None, f'Navigation to download page failed: {result.error}'
+
+				await asyncio.sleep(0.5)
+
+				# Get browser state to find download link
+				event = browser_session.event_bus.dispatch(BrowserStateRequestEvent())
+				state_result = await event.event_result()
+				assert state_result is not None
+				assert state_result.dom_state is not None
+				assert state_result.dom_state.selector_map is not None
+
+				# Find download link
+				download_link_index = None
+				for idx, element in state_result.dom_state.selector_map.items():
+					if element.attributes and element.attributes.get('id') == 'downloadLink':
+						download_link_index = idx
+						break
+
+				assert download_link_index is not None, 'Download link not found'
+
+				# Step 2: Click download link and wait for download
+				class ClickActionModel(ActionModel):
+					click_element_by_index: ClickElementAction | None = None
+
+				# Click the download link
+				result = await tools.act(
+					ClickActionModel(click_element_by_index=ClickElementAction(index=download_link_index)), browser_session
+				)
+				assert result.error is None, f'Click on download link failed: {result.error}'
+
+				# Wait for the download event
+				try:
+					download_event = await browser_session.event_bus.expect(FileDownloadedEvent, timeout=10.0)
+					downloaded_file_path = download_event.path
+				except TimeoutError:
+					pytest.fail('Download did not complete within timeout')
+
+				assert downloaded_file_path is not None, 'Downloaded file path is None'
+				assert Path(downloaded_file_path).exists(), f'Downloaded file does not exist: {downloaded_file_path}'
+
+				# Verify download is tracked by browser_session
+				assert downloaded_file_path in browser_session.downloaded_files, (
+					f'Downloaded file not tracked by browser_session: {downloaded_file_path}'
+				)
+
+				# Calculate hash of downloaded file
+				downloaded_content = Path(downloaded_file_path).read_bytes()
+				downloaded_hash = hashlib.sha256(downloaded_content).hexdigest()
+
+				print(f'✅ File downloaded: {downloaded_file_path}')
+				print(f'   Original hash: {download_upload_server.test_hash}')
+				print(f'   Downloaded hash: {downloaded_hash}')
+				assert downloaded_hash == download_upload_server.test_hash, "Downloaded file hash doesn't match original"
+
+				# Step 3: Navigate to upload page in a new tab
+				print(f'\n🔄 Opening upload page in new tab: {base_url}/upload-page')
+
+				# Debug: Check how many tabs we have before navigation
+				tabs_before = await browser_session.get_tabs()
+				print(f'📑 Tabs before navigation: {len(tabs_before)} tabs')
+				for i, tab in enumerate(tabs_before):
+					print(f'  Tab {i}: {tab.url}')
+				result = await tools.act(
+					GoToUrlActionModel(go_to_url=GoToUrlAction(url=f'{base_url}/upload-page', new_tab=True)), browser_session
+				)
+				assert result.error is None, f'Navigation to upload page failed: {result.error}'
+				print(f'✅ Navigation result: {result.extracted_content}')
+
+				# The new tab should be automatically focused after opening
+				await asyncio.sleep(2.0)  # Give more time for the new tab to load and focus
+
+				# Debug: Get all tabs
+				tabs = await browser_session.get_tabs()
+				print('\n📑 All tabs after opening upload page:')
+				for i, tab in enumerate(tabs):
+					print(f'  Tab {i}: {tab.url} - {tab.title}')
+
+				# Get browser state to find file input
+				event = browser_session.event_bus.dispatch(BrowserStateRequestEvent())
+				await event
+				state_result = await event.event_result(raise_if_any=True, raise_if_none=False)
+				assert state_result is not None
+				assert state_result.dom_state is not None
+				assert state_result.dom_state.selector_map is not None
+
+				# Debug: print page URL and title
+				print('\n🔍 Getting DOM state:')
+				print(f'  Current page URL: {state_result.url}')
+				print(f'  Current page title: {state_result.title}')
+
+				# Find file input
+				file_input_index = None
+				input_elements = []
+				for idx, element in state_result.dom_state.selector_map.items():
+					if element.tag_name and element.tag_name.lower() == 'input':
+						input_elements.append((idx, element.attributes))
+						if element.attributes and element.attributes.get('type') == 'file':
+							file_input_index = idx
+							break
+
+				print(f'Found {len(input_elements)} input elements: {input_elements}')
+				assert file_input_index is not None, 'File input not found'
+
+				# Step 4: Upload the downloaded file
+				class UploadActionModel(ActionModel):
+					upload_file_to_element: UploadFileAction | None = None
+
+				# The downloaded file should be automatically available for upload
+				result = await tools.act(
+					UploadActionModel(upload_file_to_element=UploadFileAction(index=file_input_index, path=downloaded_file_path)),
+					browser_session,
+					available_file_paths=[],  # Empty, but file is in downloaded_files
+					file_system=file_system,
+				)
+				assert result.error is None, f'File upload failed: {result.error}'
+
+				# Step 4b: Click the submit button to trigger the form submission
+				# Get browser state to find submit button
+				event = browser_session.event_bus.dispatch(BrowserStateRequestEvent())
+				state_result = await event.event_result()
+
+				# Find submit button
+				submit_button_index = None
+				if state_result and state_result.dom_state:
+					for idx, element in state_result.dom_state.selector_map.items():
+						if (
+							element.tag_name
+							and element.tag_name.lower() == 'button'
+							and element.attributes
+							and element.attributes.get('id') == 'submitButton'
+						):
+							submit_button_index = idx
+							break
+
+				assert submit_button_index is not None, 'Submit button not found'
+
+				# Click the submit button
+				result = await tools.act(
+					ClickActionModel(click_element_by_index=ClickElementAction(index=submit_button_index)), browser_session
+				)
+				assert result.error is None, f'Click on submit button failed: {result.error}'
+
+				# Wait for JavaScript to process the upload
+				await asyncio.sleep(1.0)
+
+				# Step 5: Verify upload via JavaScript (client-side hash)
+				cdp_session = await browser_session.get_or_create_cdp_session()
+
+				# Get uploaded file details from the page
+				upload_verification = await browser_session.cdp_client.send.Runtime.evaluate(
+					params={
+						'expression': """
+							(() => {
+								const fileName = document.getElementById('uploadedFileName')?.textContent;
+								const fileSize = document.getElementById('uploadedFileSize')?.textContent;
+								const fileHash = document.getElementById('uploadedFileHash')?.textContent;
+								
+								return {
+									fileName: fileName || null,
+									fileSize: fileSize || null,
+									fileHash: fileHash || null,
+									hasResult: !!document.getElementById('result').textContent.includes('successfully')
+								};
+							})()
+						""",
+						'returnByValue': True,
+					},
+					session_id=cdp_session.session_id,
+				)
+
+				upload_info = upload_verification.get('result', {}).get('value', {})
+
+				# Verify upload was successful
+				assert upload_info.get('hasResult') is True, 'Upload result not displayed'
+				assert upload_info.get('fileName') == 'test-file.txt', (
+					f'Uploaded filename mismatch: {upload_info.get("fileName")}'
+				)
+				assert upload_info.get('fileHash') == download_upload_server.test_hash, (
+					f'Uploaded file hash mismatch. Expected: {download_upload_server.test_hash}, Got: {upload_info.get("fileHash")}'
+				)
+
+				print('✅ File uploaded successfully!')
+				print(f'   Uploaded filename: {upload_info.get("fileName")}')
+				print(f'   Uploaded hash: {upload_info.get("fileHash")}')
+				print(f'   Hash matches original: {upload_info.get("fileHash") == download_upload_server.test_hash}')
+
+				# Step 6: Verify server-side upload (if needed)
+				if download_upload_server.uploaded_files:
+					server_file = download_upload_server.uploaded_files[0]
+					assert server_file['hash'] == download_upload_server.test_hash, (
+						f'Server-side hash mismatch. Expected: {download_upload_server.test_hash}, Got: {server_file["hash"]}'
+					)
+					print('✅ Server-side verification passed!')
+
+				print('\n🎉 Full circle test passed: Download → Upload with hash verification!')
+
+			finally:
+				await browser_session.stop()

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_watchdog_screenshots.py

@@ -0,0 +1,403 @@
+"""
+Test that screenshots work correctly in headless browser mode.
+"""
+
+import asyncio
+import base64
+import time
+
+import pytest
+
+from browser_use.browser import BrowserProfile, BrowserSession
+from browser_use.browser.events import NavigateToUrlEvent, ScreenshotEvent
+
+
+class TestHeadlessScreenshots:
+	"""Test screenshot functionality specifically in headless browsers"""
+
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_screenshot_works_in_headless_mode(self, httpserver):
+		"""Explicitly test that screenshots can be captured in headless=True mode"""
+		# Create a browser session with headless=True
+		browser_session = BrowserSession(
+			browser_profile=BrowserProfile(
+				headless=True,  # Explicitly set headless mode
+				user_data_dir=None,
+				keep_alive=False,
+			)
+		)
+
+		try:
+			# Start the session
+			await browser_session.start()
+			assert browser_session._cdp_client_root is not None
+
+			# Set up test page with visible content
+			httpserver.expect_request('/').respond_with_data(
+				"""<html>
+				<head><title>Headless Screenshot Test</title></head>
+				<body style="background: white; padding: 20px;">
+					<h1 style="color: black;">This is a test page</h1>
+					<p style="color: blue;">Testing screenshot capture in headless mode</p>
+					<div style="width: 200px; height: 100px; background: red;">Red Box</div>
+				</body>
+				</html>""",
+				content_type='text/html',
+			)
+
+			# Navigate to test page
+			event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))
+			await event
+			await event.event_result(raise_if_any=True, raise_if_none=False)
+
+			# Take screenshot
+			screenshot_event = browser_session.event_bus.dispatch(ScreenshotEvent())
+			await screenshot_event
+			screenshot_b64 = await screenshot_event.event_result(raise_if_any=True, raise_if_none=False)
+
+			# Verify screenshot was captured
+			assert screenshot_b64 is not None
+			assert isinstance(screenshot_b64, str)
+			assert len(screenshot_b64) > 0
+
+			# Decode and validate the screenshot
+			screenshot_bytes = base64.b64decode(screenshot_b64)
+
+			# Verify PNG signature
+			assert screenshot_bytes.startswith(b'\x89PNG\r\n\x1a\n')
+			# Should be a reasonable size (not just a blank image)
+			assert len(screenshot_bytes) > 5000, f'Screenshot too small: {len(screenshot_bytes)} bytes'
+
+			# Test full page screenshot
+			screenshot_event = browser_session.event_bus.dispatch(ScreenshotEvent(full_page=True))
+			await screenshot_event
+			full_page_screenshot = await screenshot_event.event_result(raise_if_any=True, raise_if_none=False)
+			assert full_page_screenshot is not None
+			full_page_bytes = base64.b64decode(full_page_screenshot)
+			assert full_page_bytes.startswith(b'\x89PNG\r\n\x1a\n')
+			assert len(full_page_bytes) > 5000
+
+		finally:
+			await browser_session.kill()
+
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_screenshot_with_state_summary_in_headless(self, httpserver):
+		"""Test that get_state_summary includes screenshots in headless mode"""
+		browser_session = BrowserSession(
+			browser_profile=BrowserProfile(
+				headless=True,
+				user_data_dir=None,
+				keep_alive=False,
+			)
+		)
+
+		try:
+			await browser_session.start()
+
+			# Set up test page
+			httpserver.expect_request('/').respond_with_data(
+				'<html><body><h1>State Summary Test</h1></body></html>',
+				content_type='text/html',
+			)
+			event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))
+			await event
+			await event.event_result(raise_if_any=True, raise_if_none=False)
+
+			# Get state summary
+			state = await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=False)
+
+			# Verify screenshot is included
+			assert state.screenshot is not None
+			assert isinstance(state.screenshot, str)
+			assert len(state.screenshot) > 0
+
+			# Decode and validate
+			screenshot_bytes = base64.b64decode(state.screenshot)
+			assert screenshot_bytes.startswith(b'\x89PNG\r\n\x1a\n')
+			assert len(screenshot_bytes) > 1000
+
+		finally:
+			await browser_session.kill()
+
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_screenshot_graceful_handling_in_headless(self, httpserver):
+		"""Test that screenshot handling works correctly in headless mode even with closed pages"""
+		# Set up test page
+		httpserver.expect_request('/test').respond_with_data(
+			'<html><body><h1>Test Page</h1></body></html>', content_type='text/html'
+		)
+
+		browser_session = BrowserSession(
+			browser_profile=BrowserProfile(
+				headless=True,
+				user_data_dir=None,
+				keep_alive=False,
+			)
+		)
+
+		try:
+			await browser_session.start()
+
+			# Skip complex page manipulation - CDP doesn't have direct pages access
+			pytest.skip('CDP pages access pattern needs refactoring')
+
+			# Browser should auto-create a new page on about:blank with animation
+			# With AboutBlankWatchdog, about:blank pages now have animated content, so they should have screenshots
+			state = await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=False)
+			assert state.screenshot is not None, 'Screenshot should not be None for animated about:blank pages'
+			assert state.url == 'about:blank' or state.url.startswith('chrome://'), f'Expected empty page but got {state.url}'
+
+			# Now navigate to a real page and verify screenshot works
+			event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/test')))
+			await event
+			await event.event_result(raise_if_any=True, raise_if_none=False)
+
+			# Get state with screenshot
+			state = await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=False)
+			# Should have a screenshot now
+			assert state.screenshot is not None, 'Screenshot should not be None for real pages'
+			assert isinstance(state.screenshot, str)
+			assert len(state.screenshot) > 100, 'Screenshot should have substantial content'
+			assert 'test' in state.url.lower()
+
+		finally:
+			await browser_session.kill()
+
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_parallel_screenshots_long_page(self, httpserver):
+		"""Test screenshots in a highly parallel environment with a very long page"""
+
+		# Generate a very long page (50,000px+)
+		long_content = []
+		long_content.append('<html><head><title>Very Long Page</title></head>')
+		long_content.append('<body style="margin: 0; padding: 0;">')
+
+		# Add many div elements to create a 50,000px+ long page
+		# Each div is 500px tall, so we need 100+ divs
+		for i in range(120):
+			color = f'rgb({i % 256}, {(i * 2) % 256}, {(i * 3) % 256})'
+			long_content.append(
+				f'<div style="height: 500px; background: {color}; '
+				f'display: flex; align-items: center; justify-content: center; '
+				f'font-size: 48px; color: white; text-shadow: 2px 2px 4px rgba(0,0,0,0.5);">'
+				f'Section {i + 1} - Testing Parallel Screenshots'
+				f'</div>'
+			)
+
+		long_content.append('</body></html>')
+		html_content = ''.join(long_content)
+
+		# Set up the test page
+		httpserver.expect_request('/longpage').respond_with_data(
+			html_content,
+			content_type='text/html',
+		)
+		test_url = httpserver.url_for('/longpage')
+
+		# Create 10 browser sessions
+		browser_sessions = []
+		for i in range(10):
+			session = BrowserSession(
+				browser_profile=BrowserProfile(
+					headless=True,
+					user_data_dir=None,
+					keep_alive=False,
+				)
+			)
+			browser_sessions.append(session)
+
+		try:
+			# Start all sessions sequentially to avoid playwright_global_object semaphore contention
+			# The playwright global object semaphore only allows 1 concurrent initialization
+			print('Starting 10 browser sessions sequentially...')
+			for i, session in enumerate(browser_sessions):
+				print(f'Starting session {i + 1}/10...')
+				await session.start()
+
+			# Navigate all sessions to the long page in parallel
+			print('Navigating all sessions to the long test page...')
+			await asyncio.gather(*[session.navigate(test_url) for session in browser_sessions])
+
+			# Take screenshots from all sessions
+			# Due to semaphore_limit=1, these will execute sequentially
+			print('Taking screenshots from all 10 sessions...')
+			start_time = time.time()
+			screenshot_tasks = [session.take_screenshot() for session in browser_sessions]
+
+			# Use return_exceptions=True to handle any failures gracefully
+			results = await asyncio.gather(*screenshot_tasks, return_exceptions=True)
+			total_time = time.time() - start_time
+
+			# Verify timing - with semaphore_limit=1, screenshots execute sequentially
+			# Each screenshot should take ~1.5s, so 10 × 1.5s = 15s, allow up to 30s for overhead
+			assert total_time < 30, f'Screenshots took too long: {total_time:.1f}s (should be < 30s)'
+			print(f'All screenshot attempts completed in {total_time:.1f}s')
+
+			# Separate successful screenshots from failures
+			screenshots = []
+			failures = []
+			for i, result in enumerate(results):
+				if isinstance(result, Exception):
+					failures.append((i, result))
+					print(f'Session {i} failed: {type(result).__name__}: {result}')
+				else:
+					screenshots.append(result)
+					print(f'Session {i} screenshot completed successfully')
+
+			# ALL screenshots must succeed
+			assert len(failures) == 0, (
+				f'{len(failures)} screenshots failed: {[(i, type(e).__name__, str(e)) for i, e in failures]}'
+			)
+			assert len(screenshots) == 10, f'Expected 10 successful screenshots, got {len(screenshots)}'
+			print('✅ All 10 screenshots captured successfully!')
+
+			# Verify all screenshots are valid
+			print('Verifying all 10 screenshots...')
+			for i, screenshot in enumerate(screenshots):
+				# Should not be None
+				assert screenshot is not None, f'Screenshot {i} returned None'
+				assert isinstance(screenshot, str), f'Screenshot {i} is not a string'
+				assert len(screenshot) > 0, f'Screenshot {i} is empty'
+
+				# Decode and validate
+				try:
+					screenshot_bytes = base64.b64decode(screenshot)
+				except Exception as e:
+					raise AssertionError(f'Screenshot {i} is not valid base64: {e}')
+
+				# Verify PNG signature
+				assert screenshot_bytes.startswith(b'\x89PNG\r\n\x1a\n'), f'Screenshot {i} is not a valid PNG'
+
+				# Full page screenshot should be reasonably large
+				# Due to our 6,000px height limit, expect at least 5KB
+				assert len(screenshot_bytes) > 20, f'Screenshot {i} too small: {len(screenshot_bytes)} bytes'
+				if len(screenshot_bytes) < 500:
+					print(
+						f'⚠️ Screenshot {i} failed to be taken in time, it returned a blank image instead: {len(screenshot_bytes)} bytes, perhaps the page failed to load in time?'
+					)
+
+			print('✅ All 10 screenshots validated successfully!')
+
+			# Also test taking regular (viewport) screenshots
+			print('\nTaking viewport screenshots from all sessions...')
+			start_time = time.time()
+			viewport_results = await asyncio.gather(
+				*[session.take_screenshot() for session in browser_sessions], return_exceptions=True
+			)
+			viewport_time = time.time() - start_time
+			assert viewport_time < 30, f'Viewport screenshots took too long: {viewport_time:.1f}s (should be < 30s)'
+			print(f'All viewport screenshot attempts completed in {viewport_time:.1f}s')
+
+			# Check for failures
+			viewport_screenshots = []
+			viewport_failures = []
+			for i, result in enumerate(viewport_results):
+				if isinstance(result, Exception):
+					viewport_failures.append((i, result))
+					print(f'Session {i} viewport failed: {type(result).__name__}: {result}')
+				else:
+					viewport_screenshots.append(result)
+					print(f'Session {i} viewport screenshot completed successfully')
+
+			# ALL viewport screenshots must succeed
+			assert len(viewport_failures) == 0, (
+				f'{len(viewport_failures)} viewport screenshots failed: {[(i, type(e).__name__, str(e)) for i, e in viewport_failures]}'
+			)
+			assert len(viewport_screenshots) == 10, (
+				f'Expected 10 successful viewport screenshots, got {len(viewport_screenshots)}'
+			)
+			print('✅ All 10 viewport screenshots captured successfully!')
+
+			# Verify all 10 viewport screenshots
+			print('Verifying all 10 viewport screenshots...')
+			for i, screenshot in enumerate(viewport_screenshots):
+				assert screenshot is not None, f'Viewport screenshot {i} is None'
+				screenshot_bytes = base64.b64decode(screenshot)
+				assert screenshot_bytes.startswith(b'\x89PNG\r\n\x1a\n'), f'Viewport screenshot {i} is not a valid PNG'
+				# Viewport screenshots should be reasonably sized
+				assert len(screenshot_bytes) > 10, f'Viewport screenshot {i} too small: {len(screenshot_bytes)} bytes'
+			print('✅ All 10 viewport screenshots validated successfully!')
+
+		finally:
+			# Kill all sessions in parallel
+			print('Killing all browser sessions...')
+			# Use return_exceptions=True to prevent one failed kill from affecting others
+			# This prevents "Future exception was never retrieved" errors
+			results = await asyncio.gather(*[session.kill() for session in browser_sessions], return_exceptions=True)
+
+			# Check that no exceptions were raised during cleanup
+			for i, result in enumerate(results):
+				if isinstance(result, Exception):
+					print(f'Warning: Session {i} kill raised exception: {type(result).__name__}: {result}')
+
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_screenshot_at_bottom_of_page(self, httpserver):
+		"""Test screenshot capture when scrolled to bottom of page (regression test for clipping issue)"""
+		browser_session = BrowserSession(
+			browser_profile=BrowserProfile(
+				headless=True,
+				user_data_dir=None,
+				keep_alive=False,
+			)
+		)
+
+		try:
+			await browser_session.start()
+
+			# Create a page with scrollable content
+			httpserver.expect_request('/scrollable').respond_with_data(
+				"""<html>
+				<head><title>Scrollable Page Test</title></head>
+				<body style="margin: 0; padding: 0;">
+					<div style="height: 3000px; background: linear-gradient(to bottom, red, yellow, green, blue);">
+						<div style="position: absolute; top: 0; left: 10px; font-size: 24px;">Top of page</div>
+						<div style="position: absolute; top: 50%; left: 10px; font-size: 24px;">Middle of page</div>
+						<div style="position: absolute; bottom: 10px; left: 10px; font-size: 24px;">Bottom of page</div>
+					</div>
+				</body>
+				</html>""",
+				content_type='text/html',
+			)
+
+			# Navigate to test page
+			event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/scrollable')))
+			await event
+			await event.event_result(raise_if_any=True, raise_if_none=False)
+			# Skip - get_current_tab doesn't exist in CDP session
+			pytest.skip('get_current_tab method not available in CDP session')
+
+		finally:
+			await browser_session.kill()
+
+
+class TestScreenshotEventSystem:
+	"""Tests for NEW event-driven screenshot infrastructure only."""
+
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_screenshot_response_event_dispatching(self, httpserver):
+		"""Test that ScreenshotResponseEvent is properly dispatched by event handlers."""
+		from browser_use.browser.events import ScreenshotEvent
+
+		browser_session = BrowserSession(browser_profile=BrowserProfile(headless=True, user_data_dir=None, keep_alive=False))
+
+		try:
+			await browser_session.start()
+
+			# Set up test page
+			httpserver.expect_request('/event-test').respond_with_data(
+				'<html><body><h1>Screenshot Event Test</h1></body></html>',
+				content_type='text/html',
+			)
+			event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/event-test')))
+			await event
+			await event.event_result(raise_if_any=True, raise_if_none=False)
+
+			# Test the NEW event-driven path: direct event dispatching
+			event = browser_session.event_bus.dispatch(ScreenshotEvent(full_page=False))
+			screenshot_b64 = await event.event_result()
+			assert screenshot_b64 is not None
+			assert isinstance(screenshot_b64, str)
+			assert len(base64.b64decode(screenshot_b64)) > 5000
+
+		finally:
+			await browser_session.kill()

chapter9/browser-use-rpa/browser-use/tests/ci/test_browser_watchdog_security2.py

@@ -0,0 +1,435 @@
+from browser_use.browser import BrowserProfile, BrowserSession
+
+
+class TestUrlAllowlistSecurity:
+	"""Tests for URL allowlist security bypass prevention and URL allowlist glob pattern matching."""
+
+	def test_authentication_bypass_prevention(self):
+		"""Test that the URL allowlist cannot be bypassed using authentication credentials."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		# Create a context config with a sample allowed domain
+		browser_profile = BrowserProfile(allowed_domains=['example.com'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Security vulnerability test cases
+		# These should all be detected as malicious despite containing "example.com"
+		assert watchdog._is_url_allowed('https://example.com:password@malicious.com') is False
+		assert watchdog._is_url_allowed('https://example.com@malicious.com') is False
+		assert watchdog._is_url_allowed('https://example.com%20@malicious.com') is False
+		assert watchdog._is_url_allowed('https://example.com%3A@malicious.com') is False
+
+		# Make sure legitimate auth credentials still work
+		assert watchdog._is_url_allowed('https://user:password@example.com') is True
+
+	def test_glob_pattern_matching(self):
+		"""Test that glob patterns in allowed_domains work correctly."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		# Test *.example.com pattern (should match subdomains and main domain)
+		browser_profile = BrowserProfile(allowed_domains=['*.example.com'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Should match subdomains
+		assert watchdog._is_url_allowed('https://sub.example.com') is True
+		assert watchdog._is_url_allowed('https://deep.sub.example.com') is True
+
+		# Should also match main domain
+		assert watchdog._is_url_allowed('https://example.com') is True
+
+		# Should not match other domains
+		assert watchdog._is_url_allowed('https://notexample.com') is False
+		assert watchdog._is_url_allowed('https://example.org') is False
+
+		# Test more complex glob patterns
+		browser_profile = BrowserProfile(
+			allowed_domains=[
+				'*.google.com',
+				'https://wiki.org',
+				'https://good.com',
+				'https://*.test.com',
+				'chrome://version',
+				'brave://*',
+			],
+			headless=True,
+			user_data_dir=None,
+		)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Should match domains ending with google.com
+		assert watchdog._is_url_allowed('https://google.com') is True
+		assert watchdog._is_url_allowed('https://www.google.com') is True
+		assert (
+			watchdog._is_url_allowed('https://evilgood.com') is False
+		)  # make sure we dont allow *good.com patterns, only *.good.com
+
+		# Should match domains starting with wiki
+		assert watchdog._is_url_allowed('http://wiki.org') is False
+		assert watchdog._is_url_allowed('https://wiki.org') is True
+
+		# Should not match internal domains because scheme was not provided
+		assert watchdog._is_url_allowed('chrome://google.com') is False
+		assert watchdog._is_url_allowed('chrome://abc.google.com') is False
+
+		# Test browser internal URLs
+		assert watchdog._is_url_allowed('chrome://settings') is False
+		assert watchdog._is_url_allowed('chrome://version') is True
+		assert watchdog._is_url_allowed('chrome-extension://version/') is False
+		assert watchdog._is_url_allowed('brave://anything/') is True
+		assert watchdog._is_url_allowed('about:blank') is True
+		assert watchdog._is_url_allowed('chrome://new-tab-page/') is True
+		assert watchdog._is_url_allowed('chrome://new-tab-page') is True
+
+		# Test security for glob patterns (authentication credentials bypass attempts)
+		# These should all be detected as malicious despite containing allowed domain patterns
+		assert watchdog._is_url_allowed('https://allowed.example.com:password@notallowed.com') is False
+		assert watchdog._is_url_allowed('https://subdomain.example.com@evil.com') is False
+		assert watchdog._is_url_allowed('https://sub.example.com%20@malicious.org') is False
+		assert watchdog._is_url_allowed('https://anygoogle.com@evil.org') is False
+
+		# Test pattern matching
+		assert watchdog._is_url_allowed('https://www.test.com') is True
+		assert watchdog._is_url_allowed('https://www.testx.com') is False
+
+	def test_glob_pattern_edge_cases(self):
+		"""Test edge cases for glob pattern matching to ensure proper behavior."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		# Test with domains containing glob pattern in the middle
+		browser_profile = BrowserProfile(allowed_domains=['*.google.com', 'https://wiki.org'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Verify that 'wiki*' pattern doesn't match domains that merely contain 'wiki' in the middle
+		assert watchdog._is_url_allowed('https://notawiki.com') is False
+		assert watchdog._is_url_allowed('https://havewikipages.org') is False
+		assert watchdog._is_url_allowed('https://my-wiki-site.com') is False
+
+		# Verify that '*google.com' doesn't match domains that have 'google' in the middle
+		assert watchdog._is_url_allowed('https://mygoogle.company.com') is False
+
+		# Create context with potentially risky glob pattern that demonstrates security concerns
+		browser_profile = BrowserProfile(allowed_domains=['*.google.com', '*.google.co.uk'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Should match legitimate Google domains
+		assert watchdog._is_url_allowed('https://www.google.com') is True
+		assert watchdog._is_url_allowed('https://mail.google.co.uk') is True
+
+		# Shouldn't match potentially malicious domains with a similar structure
+		# This demonstrates why the previous pattern was risky and why it's now rejected
+		assert watchdog._is_url_allowed('https://www.google.evil.com') is False
+
+	def test_automatic_www_subdomain_addition(self):
+		"""Test that root domains automatically allow www subdomain."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		# Test with simple root domains
+		browser_profile = BrowserProfile(allowed_domains=['example.com', 'test.org'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Root domain should allow itself
+		assert watchdog._is_url_allowed('https://example.com') is True
+		assert watchdog._is_url_allowed('https://test.org') is True
+
+		# Root domain should automatically allow www subdomain
+		assert watchdog._is_url_allowed('https://www.example.com') is True
+		assert watchdog._is_url_allowed('https://www.test.org') is True
+
+		# Should not allow other subdomains
+		assert watchdog._is_url_allowed('https://mail.example.com') is False
+		assert watchdog._is_url_allowed('https://sub.test.org') is False
+
+		# Should not allow unrelated domains
+		assert watchdog._is_url_allowed('https://notexample.com') is False
+		assert watchdog._is_url_allowed('https://www.notexample.com') is False
+
+	def test_www_subdomain_not_added_for_country_tlds(self):
+		"""Test www subdomain is NOT automatically added for country-specific TLDs (2+ dots)."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		# Test with country-specific TLDs - these should NOT get automatic www
+		browser_profile = BrowserProfile(
+			allowed_domains=['example.co.uk', 'test.com.au', 'site.co.jp'], headless=True, user_data_dir=None
+		)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Root domains should work exactly as specified
+		assert watchdog._is_url_allowed('https://example.co.uk') is True
+		assert watchdog._is_url_allowed('https://test.com.au') is True
+		assert watchdog._is_url_allowed('https://site.co.jp') is True
+
+		# www subdomains should NOT work automatically (user must specify explicitly)
+		assert watchdog._is_url_allowed('https://www.example.co.uk') is False
+		assert watchdog._is_url_allowed('https://www.test.com.au') is False
+		assert watchdog._is_url_allowed('https://www.site.co.jp') is False
+
+		# Other subdomains should not work
+		assert watchdog._is_url_allowed('https://mail.example.co.uk') is False
+		assert watchdog._is_url_allowed('https://api.test.com.au') is False
+
+	def test_www_subdomain_not_added_for_existing_subdomains(self):
+		"""Test that www is not automatically added for domains that already have subdomains."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		# Test with existing subdomains - should NOT get automatic www
+		browser_profile = BrowserProfile(allowed_domains=['mail.example.com', 'api.test.org'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Exact subdomain should work
+		assert watchdog._is_url_allowed('https://mail.example.com') is True
+		assert watchdog._is_url_allowed('https://api.test.org') is True
+
+		# www should NOT be automatically added to subdomains
+		assert watchdog._is_url_allowed('https://www.mail.example.com') is False
+		assert watchdog._is_url_allowed('https://www.api.test.org') is False
+
+		# Root domains should not work either
+		assert watchdog._is_url_allowed('https://example.com') is False
+		assert watchdog._is_url_allowed('https://test.org') is False
+
+	def test_www_subdomain_not_added_for_wildcard_patterns(self):
+		"""Test that www is not automatically added for wildcard patterns."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		# Test with wildcard patterns - should NOT get automatic www logic
+		browser_profile = BrowserProfile(allowed_domains=['*.example.com'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Wildcard should match everything including root and www
+		assert watchdog._is_url_allowed('https://example.com') is True
+		assert watchdog._is_url_allowed('https://www.example.com') is True
+		assert watchdog._is_url_allowed('https://mail.example.com') is True
+
+	def test_www_subdomain_not_added_for_url_patterns(self):
+		"""Test that www is not automatically added for full URL patterns."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		# Test with full URL patterns - should NOT get automatic www logic
+		browser_profile = BrowserProfile(
+			allowed_domains=['https://example.com', 'http://test.org'], headless=True, user_data_dir=None
+		)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Exact URL should work
+		assert watchdog._is_url_allowed('https://example.com/path') is True
+		assert watchdog._is_url_allowed('http://test.org/page') is True
+
+		# www should NOT be automatically added for full URL patterns
+		assert watchdog._is_url_allowed('https://www.example.com') is False
+		assert watchdog._is_url_allowed('http://www.test.org') is False
+
+	def test_is_root_domain_helper(self):
+		"""Test the _is_root_domain helper method logic."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		browser_profile = BrowserProfile(allowed_domains=['example.com'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Simple root domains (1 dot) - should return True
+		assert watchdog._is_root_domain('example.com') is True
+		assert watchdog._is_root_domain('test.org') is True
+		assert watchdog._is_root_domain('site.net') is True
+
+		# Subdomains (more than 1 dot) - should return False
+		assert watchdog._is_root_domain('www.example.com') is False
+		assert watchdog._is_root_domain('mail.example.com') is False
+		assert watchdog._is_root_domain('example.co.uk') is False
+		assert watchdog._is_root_domain('test.com.au') is False
+
+		# Wildcards - should return False
+		assert watchdog._is_root_domain('*.example.com') is False
+		assert watchdog._is_root_domain('*example.com') is False
+
+		# Full URLs - should return False
+		assert watchdog._is_root_domain('https://example.com') is False
+		assert watchdog._is_root_domain('http://test.org') is False
+
+		# Invalid domains - should return False
+		assert watchdog._is_root_domain('example') is False
+		assert watchdog._is_root_domain('') is False
+
+
+class TestUrlProhibitlistSecurity:
+	"""Tests for URL prohibitlist (blocked domains) behavior and matching semantics."""
+
+	def test_simple_prohibited_domains(self):
+		"""Domain-only patterns block exact host and www, but not other subdomains."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		browser_profile = BrowserProfile(prohibited_domains=['example.com', 'test.org'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Block exact and www
+		assert watchdog._is_url_allowed('https://example.com') is False
+		assert watchdog._is_url_allowed('https://www.example.com') is False
+		assert watchdog._is_url_allowed('https://test.org') is False
+		assert watchdog._is_url_allowed('https://www.test.org') is False
+
+		# Allow other subdomains when only root is prohibited
+		assert watchdog._is_url_allowed('https://mail.example.com') is True
+		assert watchdog._is_url_allowed('https://api.test.org') is True
+
+		# Allow unrelated domains
+		assert watchdog._is_url_allowed('https://notexample.com') is True
+
+	def test_glob_pattern_prohibited(self):
+		"""Wildcard patterns block subdomains and main domain for http/https only."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		browser_profile = BrowserProfile(prohibited_domains=['*.example.com'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Block subdomains and main domain
+		assert watchdog._is_url_allowed('https://example.com') is False
+		assert watchdog._is_url_allowed('https://www.example.com') is False
+		assert watchdog._is_url_allowed('https://mail.example.com') is False
+
+		# Allow other domains
+		assert watchdog._is_url_allowed('https://notexample.com') is True
+
+		# Wildcard with domain-only should not apply to non-http(s)
+		assert watchdog._is_url_allowed('chrome://abc.example.com') is True
+
+	def test_full_url_prohibited_patterns(self):
+		"""Full URL patterns block only matching scheme/host/prefix."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		browser_profile = BrowserProfile(prohibited_domains=['https://wiki.org', 'brave://*'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Scheme-specific blocking
+		assert watchdog._is_url_allowed('http://wiki.org') is True
+		assert watchdog._is_url_allowed('https://wiki.org') is False
+		assert watchdog._is_url_allowed('https://wiki.org/path') is False
+
+		# Internal URL prefix blocking
+		assert watchdog._is_url_allowed('brave://anything/') is False
+		assert watchdog._is_url_allowed('chrome://settings') is True
+
+	def test_internal_urls_allowed_even_when_prohibited(self):
+		"""Internal new-tab/blank URLs are always allowed regardless of prohibited list."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		browser_profile = BrowserProfile(prohibited_domains=['*'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		assert watchdog._is_url_allowed('about:blank') is True
+		assert watchdog._is_url_allowed('chrome://new-tab-page/') is True
+		assert watchdog._is_url_allowed('chrome://new-tab-page') is True
+		assert watchdog._is_url_allowed('chrome://newtab/') is True
+
+	def test_prohibited_ignored_when_allowlist_present(self):
+		"""When allowlist is set, prohibited list is ignored by design."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		browser_profile = BrowserProfile(
+			allowed_domains=['*.example.com'],
+			prohibited_domains=['https://example.com'],
+			headless=True,
+			user_data_dir=None,
+		)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Allowed by allowlist even though exact URL is in prohibited list
+		assert watchdog._is_url_allowed('https://example.com') is True
+		assert watchdog._is_url_allowed('https://www.example.com') is True
+
+		# Not in allowlist => blocked (prohibited list is not consulted in this mode)
+		assert watchdog._is_url_allowed('https://api.example.com') is True  # wildcard allowlist includes this
+		# A domain outside the allowlist should be blocked
+		assert watchdog._is_url_allowed('https://notexample.com') is False
+
+	def test_auth_credentials_do_not_cause_false_block(self):
+		"""Credentials injection with prohibited domain in username should not block unrelated hosts."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		browser_profile = BrowserProfile(prohibited_domains=['example.com'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		# Host is malicious.com, should not be blocked just because username contains example.com
+		assert watchdog._is_url_allowed('https://example.com:password@malicious.com') is True
+		assert watchdog._is_url_allowed('https://example.com@malicious.com') is True
+		assert watchdog._is_url_allowed('https://example.com%20@malicious.com') is True
+		assert watchdog._is_url_allowed('https://example.com%3A@malicious.com') is True
+
+		# Legitimate credentials to a prohibited host should be blocked
+		assert watchdog._is_url_allowed('https://user:password@example.com') is False
+
+	def test_case_insensitive_prohibited_domains(self):
+		"""Prohibited domain matching should be case-insensitive."""
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		browser_profile = BrowserProfile(prohibited_domains=['Example.COM'], headless=True, user_data_dir=None)
+		browser_session = BrowserSession(browser_profile=browser_profile)
+		event_bus = EventBus()
+		watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+		assert watchdog._is_url_allowed('https://example.com') is False
+		assert watchdog._is_url_allowed('https://WWW.EXAMPLE.COM') is False
+		assert watchdog._is_url_allowed('https://mail.example.com') is True

chapter9/browser-use-rpa/browser-use/tests/ci/test_cloud_browser.py

@@ -0,0 +1,333 @@
+"""Tests for cloud browser functionality."""
+
+import tempfile
+from pathlib import Path
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from browser_use.browser.cloud import (
+	CloudBrowserAuthError,
+	CloudBrowserClient,
+	CloudBrowserError,
+	get_cloud_browser_cdp_url,
+	stop_cloud_browser_session,
+)
+from browser_use.browser.profile import BrowserProfile
+from browser_use.browser.session import BrowserSession
+from browser_use.sync.auth import CloudAuthConfig
+
+
+@pytest.fixture
+def temp_config_dir(monkeypatch):
+	"""Create temporary config directory."""
+	with tempfile.TemporaryDirectory() as tmpdir:
+		temp_dir = Path(tmpdir) / '.config' / 'browseruse'
+		temp_dir.mkdir(parents=True, exist_ok=True)
+
+		# Use monkeypatch to set the environment variable
+		monkeypatch.setenv('BROWSER_USE_CONFIG_DIR', str(temp_dir))
+
+		yield temp_dir
+
+
+@pytest.fixture
+def mock_auth_config(temp_config_dir):
+	"""Create a mock auth config with valid token."""
+	auth_config = CloudAuthConfig(api_token='test-token', user_id='test-user-id', authorized_at=None)
+	auth_config.save_to_file()
+	return auth_config
+
+
+class TestCloudBrowserClient:
+	"""Test CloudBrowserClient class."""
+
+	async def test_create_browser_success(self, mock_auth_config):
+		"""Test successful cloud browser creation."""
+
+		# Mock response data matching the API
+		mock_response_data = {
+			'id': 'test-browser-id',
+			'status': 'active',
+			'liveUrl': 'https://live.browser-use.com?wss=test',
+			'cdpUrl': 'wss://test.proxy.daytona.works',
+			'timeoutAt': '2025-09-17T04:35:36.049892',
+			'startedAt': '2025-09-17T03:35:36.049974',
+			'finishedAt': None,
+		}
+
+		# Mock the httpx client
+		with patch('httpx.AsyncClient') as mock_client_class:
+			mock_response = AsyncMock()
+			mock_response.status_code = 201
+			mock_response.is_success = True
+			mock_response.json = lambda: mock_response_data
+
+			mock_client = AsyncMock()
+			mock_client.post.return_value = mock_response
+			mock_client_class.return_value = mock_client
+
+			client = CloudBrowserClient()
+			client.client = mock_client
+
+			result = await client.create_browser()
+
+			assert result.id == 'test-browser-id'
+			assert result.status == 'active'
+			assert result.cdpUrl == 'wss://test.proxy.daytona.works'
+
+			# Verify auth headers were included
+			mock_client.post.assert_called_once()
+			call_args = mock_client.post.call_args
+			assert 'X-Browser-Use-API-Key' in call_args.kwargs['headers']
+			assert call_args.kwargs['headers']['X-Browser-Use-API-Key'] == 'test-token'
+
+	async def test_create_browser_auth_error(self, temp_config_dir):
+		"""Test cloud browser creation with auth error."""
+
+		# Don't create auth config - should trigger auth error
+
+		with patch('httpx.AsyncClient') as mock_client_class:
+			mock_client = AsyncMock()
+			mock_client_class.return_value = mock_client
+
+			client = CloudBrowserClient()
+			client.client = mock_client
+
+			with pytest.raises(CloudBrowserAuthError) as exc_info:
+				await client.create_browser()
+
+			assert 'BROWSER_USE_API_KEY environment variable' in str(exc_info.value)
+
+	async def test_create_browser_http_401(self, mock_auth_config):
+		"""Test cloud browser creation with HTTP 401 response."""
+
+		with patch('httpx.AsyncClient') as mock_client_class:
+			mock_response = AsyncMock()
+			mock_response.status_code = 401
+			mock_response.is_success = False
+
+			mock_client = AsyncMock()
+			mock_client.post.return_value = mock_response
+			mock_client_class.return_value = mock_client
+
+			client = CloudBrowserClient()
+			client.client = mock_client
+
+			with pytest.raises(CloudBrowserAuthError) as exc_info:
+				await client.create_browser()
+
+			assert 'Authentication failed' in str(exc_info.value)
+
+	async def test_create_browser_with_env_var(self, temp_config_dir, monkeypatch):
+		"""Test cloud browser creation using BROWSER_USE_API_KEY environment variable."""
+
+		# Set environment variable
+		monkeypatch.setenv('BROWSER_USE_API_KEY', 'env-test-token')
+
+		# Mock response data matching the API
+		mock_response_data = {
+			'id': 'test-browser-id',
+			'status': 'active',
+			'liveUrl': 'https://live.browser-use.com?wss=test',
+			'cdpUrl': 'wss://test.proxy.daytona.works',
+			'timeoutAt': '2025-09-17T04:35:36.049892',
+			'startedAt': '2025-09-17T03:35:36.049974',
+			'finishedAt': None,
+		}
+
+		with patch('httpx.AsyncClient') as mock_client_class:
+			mock_response = AsyncMock()
+			mock_response.status_code = 201
+			mock_response.is_success = True
+			mock_response.json = lambda: mock_response_data
+
+			mock_client = AsyncMock()
+			mock_client.post.return_value = mock_response
+			mock_client_class.return_value = mock_client
+
+			client = CloudBrowserClient()
+			client.client = mock_client
+
+			result = await client.create_browser()
+
+			assert result.id == 'test-browser-id'
+			assert result.status == 'active'
+			assert result.cdpUrl == 'wss://test.proxy.daytona.works'
+
+			# Verify environment variable was used
+			mock_client.post.assert_called_once()
+			call_args = mock_client.post.call_args
+			assert 'X-Browser-Use-API-Key' in call_args.kwargs['headers']
+			assert call_args.kwargs['headers']['X-Browser-Use-API-Key'] == 'env-test-token'
+
+	async def test_stop_browser_success(self, mock_auth_config):
+		"""Test successful cloud browser session stop."""
+
+		# Mock response data for stop
+		mock_response_data = {
+			'id': 'test-browser-id',
+			'status': 'stopped',
+			'liveUrl': 'https://live.browser-use.com?wss=test',
+			'cdpUrl': 'wss://test.proxy.daytona.works',
+			'timeoutAt': '2025-09-17T04:35:36.049892',
+			'startedAt': '2025-09-17T03:35:36.049974',
+			'finishedAt': '2025-09-17T04:35:36.049892',
+		}
+
+		with patch('httpx.AsyncClient') as mock_client_class:
+			mock_response = AsyncMock()
+			mock_response.status_code = 200
+			mock_response.is_success = True
+			mock_response.json = lambda: mock_response_data
+
+			mock_client = AsyncMock()
+			mock_client.patch.return_value = mock_response
+			mock_client_class.return_value = mock_client
+
+			client = CloudBrowserClient()
+			client.client = mock_client
+			client.current_session_id = 'test-browser-id'
+
+			result = await client.stop_browser()
+
+			assert result.id == 'test-browser-id'
+			assert result.status == 'stopped'
+			assert result.finishedAt is not None
+
+			# Verify correct API call
+			mock_client.patch.assert_called_once()
+			call_args = mock_client.patch.call_args
+			assert 'test-browser-id' in call_args.args[0]  # URL contains session ID
+			assert call_args.kwargs['json'] == {'action': 'stop'}
+			assert 'X-Browser-Use-API-Key' in call_args.kwargs['headers']
+
+	async def test_stop_browser_session_not_found(self, mock_auth_config):
+		"""Test stopping a browser session that doesn't exist."""
+
+		with patch('httpx.AsyncClient') as mock_client_class:
+			mock_response = AsyncMock()
+			mock_response.status_code = 404
+			mock_response.is_success = False
+
+			mock_client = AsyncMock()
+			mock_client.patch.return_value = mock_response
+			mock_client_class.return_value = mock_client
+
+			client = CloudBrowserClient()
+			client.client = mock_client
+
+			with pytest.raises(CloudBrowserError) as exc_info:
+				await client.stop_browser('nonexistent-session')
+
+			assert 'not found' in str(exc_info.value)
+
+
+class TestBrowserSessionCloudIntegration:
+	"""Test BrowserSession integration with cloud browsers."""
+
+	async def test_cloud_browser_profile_property(self):
+		"""Test that cloud_browser property works correctly."""
+
+		profile = BrowserProfile(use_cloud=True)
+		session = BrowserSession(browser_profile=profile)
+
+		assert session.cloud_browser is True
+		assert session.browser_profile.use_cloud is True
+
+	async def test_browser_session_cloud_browser_logic(self, mock_auth_config):
+		"""Test that cloud browser profile settings work correctly."""
+
+		# Test cloud browser profile creation
+		profile = BrowserProfile(use_cloud=True)
+		assert profile.use_cloud is True
+
+		# Test that BrowserSession respects cloud_browser setting
+		session = BrowserSession(browser_profile=profile)
+		assert session.cloud_browser is True
+
+		# Test that get_cloud_browser_cdp_url works with mocked API
+		with patch('browser_use.browser.cloud.get_cloud_browser_cdp_url') as mock_get_cdp_url:
+			mock_get_cdp_url.return_value = 'wss://test.proxy.daytona.works'
+
+			cdp_url = await mock_get_cdp_url()
+			assert cdp_url == 'wss://test.proxy.daytona.works'
+			mock_get_cdp_url.assert_called_once()
+
+
+async def test_get_cloud_browser_cdp_url_function(mock_auth_config):
+	"""Test the get_cloud_browser_cdp_url convenience function."""
+
+	mock_response_data = {
+		'id': 'test-browser-id',
+		'status': 'active',
+		'liveUrl': 'https://live.browser-use.com?wss=test',
+		'cdpUrl': 'wss://test.proxy.daytona.works',
+		'timeoutAt': '2025-09-17T04:35:36.049892',
+		'startedAt': '2025-09-17T03:35:36.049974',
+		'finishedAt': None,
+	}
+
+	with patch('httpx.AsyncClient') as mock_client_class:
+		mock_response = AsyncMock()
+		mock_response.status_code = 201
+		mock_response.is_success = True
+		mock_response.json = lambda: mock_response_data
+
+		mock_client = AsyncMock()
+		mock_client.post.return_value = mock_response
+		mock_client_class.return_value = mock_client
+
+		cdp_url = await get_cloud_browser_cdp_url()
+
+		assert cdp_url == 'wss://test.proxy.daytona.works'
+
+
+async def test_cloud_browser_auth_error_no_fallback(temp_config_dir):
+	"""Test that cloud browser throws error when auth fails (no fallback)."""
+
+	# Don't create auth config to trigger auth error
+	profile = BrowserProfile(use_cloud=True)
+
+	# Test that cloud browser client raises error without fallback
+	with patch('browser_use.browser.cloud.get_cloud_browser_cdp_url') as mock_cloud_cdp:
+		mock_cloud_cdp.side_effect = CloudBrowserAuthError('No auth token')
+
+		# Verify that the cloud browser client raises the expected error
+		with pytest.raises(CloudBrowserAuthError) as exc_info:
+			await get_cloud_browser_cdp_url()
+
+		assert 'BROWSER_USE_API_KEY environment variable' in str(exc_info.value)
+
+		# Verify profile state unchanged (no fallback)
+		assert profile.use_cloud is True
+		assert profile.is_local is False
+
+
+async def test_stop_cloud_browser_session_function(mock_auth_config):
+	"""Test the stop_cloud_browser_session convenience function."""
+
+	mock_response_data = {
+		'id': 'test-browser-id',
+		'status': 'stopped',
+		'liveUrl': 'https://live.browser-use.com?wss=test',
+		'cdpUrl': 'wss://test.proxy.daytona.works',
+		'timeoutAt': '2025-09-17T04:35:36.049892',
+		'startedAt': '2025-09-17T03:35:36.049974',
+		'finishedAt': '2025-09-17T04:35:36.049892',
+	}
+
+	with patch('httpx.AsyncClient') as mock_client_class:
+		mock_response = AsyncMock()
+		mock_response.status_code = 200
+		mock_response.is_success = True
+		mock_response.json = lambda: mock_response_data
+
+		mock_client = AsyncMock()
+		mock_client.patch.return_value = mock_response
+		mock_client_class.return_value = mock_client
+
+		result = await stop_cloud_browser_session('test-browser-id')
+
+		assert result.id == 'test-browser-id'
+		assert result.status == 'stopped'

chapter9/browser-use-rpa/browser-use/tests/ci/test_config.py

@@ -0,0 +1,120 @@
+"""Tests for lazy loading configuration system."""
+
+import os
+
+from browser_use.config import CONFIG
+
+
+class TestLazyConfig:
+	"""Test lazy loading of environment variables through CONFIG object."""
+
+	def test_config_reads_env_vars_lazily(self):
+		"""Test that CONFIG reads environment variables each time they're accessed."""
+		# Set an env var
+		original_value = os.environ.get('BROWSER_USE_LOGGING_LEVEL', '')
+		try:
+			os.environ['BROWSER_USE_LOGGING_LEVEL'] = 'debug'
+			assert CONFIG.BROWSER_USE_LOGGING_LEVEL == 'debug'
+
+			# Change the env var
+			os.environ['BROWSER_USE_LOGGING_LEVEL'] = 'info'
+			assert CONFIG.BROWSER_USE_LOGGING_LEVEL == 'info'
+
+			# Delete the env var to test default
+			del os.environ['BROWSER_USE_LOGGING_LEVEL']
+			assert CONFIG.BROWSER_USE_LOGGING_LEVEL == 'info'  # default value
+		finally:
+			# Restore original value
+			if original_value:
+				os.environ['BROWSER_USE_LOGGING_LEVEL'] = original_value
+			else:
+				os.environ.pop('BROWSER_USE_LOGGING_LEVEL', None)
+
+	def test_boolean_env_vars(self):
+		"""Test boolean environment variables are parsed correctly."""
+		original_value = os.environ.get('ANONYMIZED_TELEMETRY', '')
+		try:
+			# Test true values
+			for true_val in ['true', 'True', 'TRUE', 'yes', 'Yes', '1']:
+				os.environ['ANONYMIZED_TELEMETRY'] = true_val
+				assert CONFIG.ANONYMIZED_TELEMETRY is True, f'Failed for value: {true_val}'
+
+			# Test false values
+			for false_val in ['false', 'False', 'FALSE', 'no', 'No', '0']:
+				os.environ['ANONYMIZED_TELEMETRY'] = false_val
+				assert CONFIG.ANONYMIZED_TELEMETRY is False, f'Failed for value: {false_val}'
+		finally:
+			if original_value:
+				os.environ['ANONYMIZED_TELEMETRY'] = original_value
+			else:
+				os.environ.pop('ANONYMIZED_TELEMETRY', None)
+
+	def test_api_keys_lazy_loading(self):
+		"""Test API keys are loaded lazily."""
+		original_value = os.environ.get('OPENAI_API_KEY', '')
+		try:
+			# Test empty default
+			os.environ.pop('OPENAI_API_KEY', None)
+			assert CONFIG.OPENAI_API_KEY == ''
+
+			# Set a value
+			os.environ['OPENAI_API_KEY'] = 'test-key-123'
+			assert CONFIG.OPENAI_API_KEY == 'test-key-123'
+
+			# Change the value
+			os.environ['OPENAI_API_KEY'] = 'new-key-456'
+			assert CONFIG.OPENAI_API_KEY == 'new-key-456'
+		finally:
+			if original_value:
+				os.environ['OPENAI_API_KEY'] = original_value
+			else:
+				os.environ.pop('OPENAI_API_KEY', None)
+
+	def test_path_configuration(self):
+		"""Test path configuration variables."""
+		original_value = os.environ.get('XDG_CACHE_HOME', '')
+		try:
+			# Test custom path
+			test_path = '/tmp/test-cache'
+			os.environ['XDG_CACHE_HOME'] = test_path
+			# Use Path().resolve() to handle symlinks (e.g., /tmp -> /private/tmp on macOS)
+			from pathlib import Path
+
+			assert CONFIG.XDG_CACHE_HOME == Path(test_path).resolve()
+
+			# Test default path expansion
+			os.environ.pop('XDG_CACHE_HOME', None)
+			assert '/.cache' in str(CONFIG.XDG_CACHE_HOME)
+		finally:
+			if original_value:
+				os.environ['XDG_CACHE_HOME'] = original_value
+			else:
+				os.environ.pop('XDG_CACHE_HOME', None)
+
+	def test_cloud_sync_inherits_telemetry(self):
+		"""Test BROWSER_USE_CLOUD_SYNC inherits from ANONYMIZED_TELEMETRY when not set."""
+		telemetry_original = os.environ.get('ANONYMIZED_TELEMETRY', '')
+		sync_original = os.environ.get('BROWSER_USE_CLOUD_SYNC', '')
+		try:
+			# When BROWSER_USE_CLOUD_SYNC is not set, it should inherit from ANONYMIZED_TELEMETRY
+			os.environ['ANONYMIZED_TELEMETRY'] = 'true'
+			os.environ.pop('BROWSER_USE_CLOUD_SYNC', None)
+			assert CONFIG.BROWSER_USE_CLOUD_SYNC is True
+
+			os.environ['ANONYMIZED_TELEMETRY'] = 'false'
+			os.environ.pop('BROWSER_USE_CLOUD_SYNC', None)
+			assert CONFIG.BROWSER_USE_CLOUD_SYNC is False
+
+			# When explicitly set, it should use its own value
+			os.environ['ANONYMIZED_TELEMETRY'] = 'false'
+			os.environ['BROWSER_USE_CLOUD_SYNC'] = 'true'
+			assert CONFIG.BROWSER_USE_CLOUD_SYNC is True
+		finally:
+			if telemetry_original:
+				os.environ['ANONYMIZED_TELEMETRY'] = telemetry_original
+			else:
+				os.environ.pop('ANONYMIZED_TELEMETRY', None)
+			if sync_original:
+				os.environ['BROWSER_USE_CLOUD_SYNC'] = sync_original
+			else:
+				os.environ.pop('BROWSER_USE_CLOUD_SYNC', None)

chapter9/browser-use-rpa/browser-use/tests/ci/test_filesystem.py

@@ -0,0 +1,937 @@
+"""Tests for the FileSystem class and related file operations."""
+
+import asyncio
+import tempfile
+from pathlib import Path
+
+import pytest
+
+from browser_use.filesystem.file_system import (
+	DEFAULT_FILE_SYSTEM_PATH,
+	INVALID_FILENAME_ERROR_MESSAGE,
+	CsvFile,
+	FileSystem,
+	FileSystemState,
+	JsonFile,
+	MarkdownFile,
+	TxtFile,
+)
+
+
+class TestBaseFile:
+	"""Test the BaseFile abstract base class and its implementations."""
+
+	def test_markdown_file_creation(self):
+		"""Test MarkdownFile creation and basic properties."""
+		md_file = MarkdownFile(name='test', content='# Hello World')
+
+		assert md_file.name == 'test'
+		assert md_file.content == '# Hello World'
+		assert md_file.extension == 'md'
+		assert md_file.full_name == 'test.md'
+		assert md_file.get_size == 13
+		assert md_file.get_line_count == 1
+
+	def test_txt_file_creation(self):
+		"""Test TxtFile creation and basic properties."""
+		txt_file = TxtFile(name='notes', content='Hello\nWorld')
+
+		assert txt_file.name == 'notes'
+		assert txt_file.content == 'Hello\nWorld'
+		assert txt_file.extension == 'txt'
+		assert txt_file.full_name == 'notes.txt'
+		assert txt_file.get_size == 11
+		assert txt_file.get_line_count == 2
+
+	def test_json_file_creation(self):
+		"""Test JsonFile creation and basic properties."""
+		json_content = '{"name": "John", "age": 30, "city": "New York"}'
+		json_file = JsonFile(name='data', content=json_content)
+
+		assert json_file.name == 'data'
+		assert json_file.content == json_content
+		assert json_file.extension == 'json'
+		assert json_file.full_name == 'data.json'
+		assert json_file.get_size == len(json_content)
+		assert json_file.get_line_count == 1
+
+	def test_csv_file_creation(self):
+		"""Test CsvFile creation and basic properties."""
+		csv_content = 'name,age,city\nJohn,30,New York\nJane,25,London'
+		csv_file = CsvFile(name='users', content=csv_content)
+
+		assert csv_file.name == 'users'
+		assert csv_file.content == csv_content
+		assert csv_file.extension == 'csv'
+		assert csv_file.full_name == 'users.csv'
+		assert csv_file.get_size == len(csv_content)
+		assert csv_file.get_line_count == 3
+
+	def test_file_content_operations(self):
+		"""Test content update and append operations."""
+		file_obj = TxtFile(name='test')
+
+		# Initial content
+		assert file_obj.content == ''
+		assert file_obj.get_size == 0
+
+		# Write content
+		file_obj.write_file_content('First line')
+		assert file_obj.content == 'First line'
+		assert file_obj.get_size == 10
+
+		# Append content
+		file_obj.append_file_content('\nSecond line')
+		assert file_obj.content == 'First line\nSecond line'
+		assert file_obj.get_line_count == 2
+
+		# Update content
+		file_obj.update_content('New content')
+		assert file_obj.content == 'New content'
+
+	async def test_file_disk_operations(self):
+		"""Test file sync to disk operations."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			tmp_path = Path(tmp_dir)
+			file_obj = MarkdownFile(name='test', content='# Test Content')
+
+			# Test sync to disk
+			await file_obj.sync_to_disk(tmp_path)
+
+			# Verify file was created on disk
+			file_path = tmp_path / 'test.md'
+			assert file_path.exists()
+			assert file_path.read_text() == '# Test Content'
+
+			# Test write operation
+			await file_obj.write('# New Content', tmp_path)
+			assert file_path.read_text() == '# New Content'
+			assert file_obj.content == '# New Content'
+
+			# Test append operation
+			await file_obj.append('\n## Section 2', tmp_path)
+			expected_content = '# New Content\n## Section 2'
+			assert file_path.read_text() == expected_content
+			assert file_obj.content == expected_content
+
+	async def test_json_file_disk_operations(self):
+		"""Test JSON file sync to disk operations."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			tmp_path = Path(tmp_dir)
+			json_content = '{"users": [{"name": "John", "age": 30}]}'
+			json_file = JsonFile(name='data', content=json_content)
+
+			# Test sync to disk
+			await json_file.sync_to_disk(tmp_path)
+
+			# Verify file was created on disk
+			file_path = tmp_path / 'data.json'
+			assert file_path.exists()
+			assert file_path.read_text() == json_content
+
+			# Test write operation
+			new_content = '{"users": [{"name": "Jane", "age": 25}]}'
+			await json_file.write(new_content, tmp_path)
+			assert file_path.read_text() == new_content
+			assert json_file.content == new_content
+
+			# Test append operation
+			await json_file.append(', {"name": "Bob", "age": 35}', tmp_path)
+			expected_content = new_content + ', {"name": "Bob", "age": 35}'
+			assert file_path.read_text() == expected_content
+			assert json_file.content == expected_content
+
+	async def test_csv_file_disk_operations(self):
+		"""Test CSV file sync to disk operations."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			tmp_path = Path(tmp_dir)
+			csv_content = 'name,age,city\nJohn,30,New York'
+			csv_file = CsvFile(name='users', content=csv_content)
+
+			# Test sync to disk
+			await csv_file.sync_to_disk(tmp_path)
+
+			# Verify file was created on disk
+			file_path = tmp_path / 'users.csv'
+			assert file_path.exists()
+			assert file_path.read_text() == csv_content
+
+			# Test write operation
+			new_content = 'name,age,city\nJane,25,London'
+			await csv_file.write(new_content, tmp_path)
+			assert file_path.read_text() == new_content
+			assert csv_file.content == new_content
+
+			# Test append operation
+			await csv_file.append('\nBob,35,Paris', tmp_path)
+			expected_content = new_content + '\nBob,35,Paris'
+			assert file_path.read_text() == expected_content
+			assert csv_file.content == expected_content
+
+	def test_file_sync_to_disk_sync(self):
+		"""Test synchronous disk sync operation."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			tmp_path = Path(tmp_dir)
+			file_obj = TxtFile(name='sync_test', content='Sync content')
+
+			# Test synchronous sync
+			file_obj.sync_to_disk_sync(tmp_path)
+
+			# Verify file was created
+			file_path = tmp_path / 'sync_test.txt'
+			assert file_path.exists()
+			assert file_path.read_text() == 'Sync content'
+
+
+class TestFileSystem:
+	"""Test the FileSystem class functionality."""
+
+	@pytest.fixture
+	def temp_filesystem(self):
+		"""Create a temporary FileSystem for testing."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			fs = FileSystem(base_dir=tmp_dir, create_default_files=True)
+			yield fs
+			try:
+				fs.nuke()
+			except Exception:
+				pass  # Directory might already be cleaned up
+
+	@pytest.fixture
+	def empty_filesystem(self):
+		"""Create a temporary FileSystem without default files."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			fs = FileSystem(base_dir=tmp_dir, create_default_files=False)
+			yield fs
+			try:
+				fs.nuke()
+			except Exception:
+				pass
+
+	def test_filesystem_initialization(self, temp_filesystem):
+		"""Test FileSystem initialization with default files."""
+		fs = temp_filesystem
+
+		# Check that base directory and data directory exist
+		assert fs.base_dir.exists()
+		assert fs.data_dir.exists()
+		assert fs.data_dir.name == DEFAULT_FILE_SYSTEM_PATH
+
+		# Check default files are created
+		assert 'todo.md' in fs.files
+		assert len(fs.files) == 1
+
+		# Check files exist on disk
+		todo_path = fs.data_dir / 'todo.md'
+		assert todo_path.exists()
+
+	def test_filesystem_without_default_files(self, empty_filesystem):
+		"""Test FileSystem initialization without default files."""
+		fs = empty_filesystem
+
+		assert fs.base_dir.exists()
+		assert fs.data_dir.exists()
+		assert len(fs.files) == 0
+
+	def test_get_allowed_extensions(self, temp_filesystem):
+		"""Test getting allowed file extensions."""
+		fs = temp_filesystem
+		extensions = fs.get_allowed_extensions()
+
+		assert 'md' in extensions
+		assert 'txt' in extensions
+		assert 'json' in extensions
+		assert 'csv' in extensions
+
+	def test_filename_validation(self, temp_filesystem):
+		"""Test filename validation."""
+		fs = temp_filesystem
+
+		# Valid filenames
+		assert fs._is_valid_filename('test.md') is True
+		assert fs._is_valid_filename('my_file.txt') is True
+		assert fs._is_valid_filename('file-name.md') is True
+		assert fs._is_valid_filename('file123.txt') is True
+		assert fs._is_valid_filename('data.json') is True
+		assert fs._is_valid_filename('users.csv') is True
+
+		# Invalid filenames
+		assert fs._is_valid_filename('test.doc') is False  # wrong extension
+		assert fs._is_valid_filename('test') is False  # no extension
+		assert fs._is_valid_filename('test.md.txt') is False  # multiple extensions
+		assert fs._is_valid_filename('test with spaces.md') is False  # spaces
+		assert fs._is_valid_filename('test@file.md') is False  # special chars
+		assert fs._is_valid_filename('.md') is False  # no name
+		assert fs._is_valid_filename('.json') is False  # no name
+		assert fs._is_valid_filename('.csv') is False  # no name
+
+	def test_filename_parsing(self, temp_filesystem):
+		"""Test filename parsing into name and extension."""
+		fs = temp_filesystem
+
+		name, ext = fs._parse_filename('test.md')
+		assert name == 'test'
+		assert ext == 'md'
+
+		name, ext = fs._parse_filename('my_file.TXT')
+		assert name == 'my_file'
+		assert ext == 'txt'  # Should be lowercased
+
+		name, ext = fs._parse_filename('data.json')
+		assert name == 'data'
+		assert ext == 'json'
+
+		name, ext = fs._parse_filename('users.CSV')
+		assert name == 'users'
+		assert ext == 'csv'  # Should be lowercased
+
+	def test_get_file(self, temp_filesystem):
+		"""Test getting files from the filesystem."""
+		fs = temp_filesystem
+
+		# Get non-existent file
+		non_existent = fs.get_file('nonexistent.md')
+		assert non_existent is None
+
+		# Get file with invalid name
+		invalid = fs.get_file('invalid@name.md')
+		assert invalid is None
+
+	def test_list_files(self, temp_filesystem):
+		"""Test listing files in the filesystem."""
+		fs = temp_filesystem
+		files = fs.list_files()
+
+		assert 'todo.md' in files
+		assert len(files) == 1
+
+	def test_display_file(self, temp_filesystem):
+		"""Test displaying file content."""
+		fs = temp_filesystem
+
+		# Display existing file
+		content = fs.display_file('todo.md')
+		assert content == ''  # Default files are empty
+
+		# Display non-existent file
+		content = fs.display_file('nonexistent.md')
+		assert content is None
+
+		# Display file with invalid name
+		content = fs.display_file('invalid@name.md')
+		assert content is None
+
+	async def test_read_file(self, temp_filesystem: FileSystem):
+		"""Test reading file content with proper formatting."""
+		fs: FileSystem = temp_filesystem
+
+		# Read existing empty file
+		result = await fs.read_file('todo.md')
+		expected = 'Read from file todo.md.\n<content>\n\n</content>'
+		assert result == expected
+
+		# Read non-existent file
+		result = await fs.read_file('nonexistent.md')
+		assert result == "File 'nonexistent.md' not found."
+
+		# Read file with invalid name
+		result = await fs.read_file('invalid@name.md')
+		assert result == INVALID_FILENAME_ERROR_MESSAGE
+
+	async def test_write_file(self, temp_filesystem):
+		"""Test writing content to files."""
+		fs = temp_filesystem
+
+		# Write to existing file
+		result = await fs.write_file('results.md', '# Test Results\nThis is a test.')
+		assert result == 'Data written to file results.md successfully.'
+
+		# Verify content was written
+		content = await fs.read_file('results.md')
+		assert '# Test Results\nThis is a test.' in content
+
+		# Write to new file
+		result = await fs.write_file('new_file.txt', 'New file content')
+		assert result == 'Data written to file new_file.txt successfully.'
+		assert 'new_file.txt' in fs.files
+		assert fs.get_file('new_file.txt').content == 'New file content'
+
+		# Write with invalid filename
+		result = await fs.write_file('invalid@name.md', 'content')
+		assert result == INVALID_FILENAME_ERROR_MESSAGE
+
+		# Write with invalid extension
+		result = await fs.write_file('test.doc', 'content')
+		assert result == INVALID_FILENAME_ERROR_MESSAGE
+
+	async def test_write_json_file(self, temp_filesystem):
+		"""Test writing JSON files."""
+		fs = temp_filesystem
+
+		# Write valid JSON content
+		json_content = '{"users": [{"name": "John", "age": 30}, {"name": "Jane", "age": 25}]}'
+		result = await fs.write_file('data.json', json_content)
+		assert result == 'Data written to file data.json successfully.'
+
+		# Verify content was written
+		content = await fs.read_file('data.json')
+		assert json_content in content
+
+		# Verify file object was created
+		assert 'data.json' in fs.files
+		file_obj = fs.get_file('data.json')
+		assert file_obj is not None
+		assert isinstance(file_obj, JsonFile)
+		assert file_obj.content == json_content
+
+		# Write to new JSON file
+		result = await fs.write_file('config.json', '{"debug": true, "port": 4242}')
+		assert result == 'Data written to file config.json successfully.'
+		assert 'config.json' in fs.files
+
+	async def test_write_csv_file(self, temp_filesystem):
+		"""Test writing CSV files."""
+		fs = temp_filesystem
+
+		# Write valid CSV content
+		csv_content = 'name,age,city\nJohn,30,New York\nJane,25,London\nBob,35,Paris'
+		result = await fs.write_file('users.csv', csv_content)
+		assert result == 'Data written to file users.csv successfully.'
+
+		# Verify content was written
+		content = await fs.read_file('users.csv')
+		assert csv_content in content
+
+		# Verify file object was created
+		assert 'users.csv' in fs.files
+		file_obj = fs.get_file('users.csv')
+		assert file_obj is not None
+		assert isinstance(file_obj, CsvFile)
+		assert file_obj.content == csv_content
+
+		# Write to new CSV file
+		result = await fs.write_file('products.csv', 'id,name,price\n1,Laptop,999.99\n2,Mouse,29.99')
+		assert result == 'Data written to file products.csv successfully.'
+		assert 'products.csv' in fs.files
+
+	async def test_append_file(self, temp_filesystem):
+		"""Test appending content to files."""
+		fs = temp_filesystem
+
+		# First write some content
+		await fs.write_file('test.md', '# Title')
+
+		# Append content
+		result = await fs.append_file('test.md', '\n## Section 1')
+		assert result == 'Data appended to file test.md successfully.'
+
+		# Verify content was appended
+		content = fs.get_file('test.md').content
+		assert content == '# Title\n## Section 1'
+
+		# Append to non-existent file
+		result = await fs.append_file('nonexistent.md', 'content')
+		assert result == "File 'nonexistent.md' not found."
+
+		# Append with invalid filename
+		result = await fs.append_file('invalid@name.md', 'content')
+		assert result == INVALID_FILENAME_ERROR_MESSAGE
+
+	async def test_append_json_file(self, temp_filesystem):
+		"""Test appending content to JSON files."""
+		fs = temp_filesystem
+
+		# First write some JSON content
+		await fs.write_file('data.json', '{"users": [{"name": "John", "age": 30}]}')
+
+		# Append additional JSON content (note: this creates invalid JSON, but tests the append functionality)
+		result = await fs.append_file('data.json', ', {"name": "Jane", "age": 25}')
+		assert result == 'Data appended to file data.json successfully.'
+
+		# Verify content was appended
+		file_obj = fs.get_file('data.json')
+		assert file_obj is not None
+		expected_content = '{"users": [{"name": "John", "age": 30}]}, {"name": "Jane", "age": 25}'
+		assert file_obj.content == expected_content
+
+	async def test_append_csv_file(self, temp_filesystem):
+		"""Test appending content to CSV files."""
+		fs = temp_filesystem
+
+		# First write some CSV content
+		await fs.write_file('users.csv', 'name,age,city\nJohn,30,New York')
+
+		# Append additional CSV row
+		result = await fs.append_file('users.csv', '\nJane,25,London')
+		assert result == 'Data appended to file users.csv successfully.'
+
+		# Verify content was appended
+		file_obj = fs.get_file('users.csv')
+		assert file_obj is not None
+		expected_content = 'name,age,city\nJohn,30,New York\nJane,25,London'
+		assert file_obj.content == expected_content
+
+		# Append another row
+		await fs.append_file('users.csv', '\nBob,35,Paris')
+		expected_content = 'name,age,city\nJohn,30,New York\nJane,25,London\nBob,35,Paris'
+		assert file_obj.content == expected_content
+
+	async def test_save_extracted_content(self, temp_filesystem):
+		"""Test saving extracted content with auto-numbering."""
+		fs = temp_filesystem
+
+		# Save first extracted content
+		result = await fs.save_extracted_content('First extracted content')
+		assert result == 'Extracted content saved to file extracted_content_0.md successfully.'
+		assert 'extracted_content_0.md' in fs.files
+		assert fs.extracted_content_count == 1
+
+		# Save second extracted content
+		result = await fs.save_extracted_content('Second extracted content')
+		assert result == 'Extracted content saved to file extracted_content_1.md successfully.'
+		assert 'extracted_content_1.md' in fs.files
+		assert fs.extracted_content_count == 2
+
+		# Verify content
+		content1 = fs.get_file('extracted_content_0.md').content
+		content2 = fs.get_file('extracted_content_1.md').content
+		assert content1 == 'First extracted content'
+		assert content2 == 'Second extracted content'
+
+	async def test_describe_with_content(self, temp_filesystem):
+		"""Test describing filesystem with files containing content."""
+		fs = temp_filesystem
+
+		# Add content to files
+		await fs.write_file('results.md', '# Results\nTest results here.')
+		await fs.write_file('notes.txt', 'These are my notes.')
+
+		description = fs.describe()
+
+		# Should contain file information
+		assert 'results.md' in description
+		assert 'notes.txt' in description
+		assert '# Results' in description
+		assert 'These are my notes.' in description
+		assert 'lines' in description
+
+	async def test_describe_large_files(self, temp_filesystem):
+		"""Test describing filesystem with large files (truncated content)."""
+		fs = temp_filesystem
+
+		# Create a large file
+		large_content = '\n'.join([f'Line {i}' for i in range(100)])
+		await fs.write_file('large.md', large_content)
+
+		description = fs.describe()
+
+		# Should be truncated with "more lines" indicator
+		assert 'large.md' in description
+		assert 'more lines' in description
+		assert 'Line 0' in description  # Start should be shown
+		assert 'Line 99' in description  # End should be shown
+
+	def test_get_todo_contents(self, temp_filesystem):
+		"""Test getting todo file contents."""
+		fs = temp_filesystem
+
+		# Initially empty
+		todo_content = fs.get_todo_contents()
+		assert todo_content == ''
+
+		# Add content to todo
+		fs.get_file('todo.md').update_content('- [ ] Task 1\n- [ ] Task 2')
+		todo_content = fs.get_todo_contents()
+		assert '- [ ] Task 1' in todo_content
+
+	def test_get_state(self, temp_filesystem):
+		"""Test getting filesystem state."""
+		fs = temp_filesystem
+
+		state = fs.get_state()
+
+		assert isinstance(state, FileSystemState)
+		assert state.base_dir == str(fs.base_dir)
+		assert state.extracted_content_count == 0
+		assert 'todo.md' in state.files
+
+	async def test_from_state(self, temp_filesystem):
+		"""Test restoring filesystem from state."""
+		fs = temp_filesystem
+
+		# Add some content
+		await fs.write_file('results.md', '# Original Results')
+		await fs.write_file('custom.txt', 'Custom content')
+		await fs.save_extracted_content('Extracted data')
+
+		# Get state
+		state = fs.get_state()
+
+		# Create new filesystem from state
+		fs2 = FileSystem.from_state(state)
+
+		# Verify restoration
+		assert fs2.base_dir == fs.base_dir
+		assert fs2.extracted_content_count == fs.extracted_content_count
+		assert len(fs2.files) == len(fs.files)
+
+		# Verify file contents
+		file_obj = fs2.get_file('results.md')
+		assert file_obj is not None
+		assert file_obj.content == '# Original Results'
+		file_obj = fs2.get_file('custom.txt')
+		assert file_obj is not None
+		assert file_obj.content == 'Custom content'
+		file_obj = fs2.get_file('extracted_content_0.md')
+		assert file_obj is not None
+		assert file_obj.content == 'Extracted data'
+
+		# Verify files exist on disk
+		assert (fs2.data_dir / 'results.md').exists()
+		assert (fs2.data_dir / 'custom.txt').exists()
+		assert (fs2.data_dir / 'extracted_content_0.md').exists()
+
+		# Clean up second filesystem
+		fs2.nuke()
+
+	async def test_complete_workflow_with_json_csv(self):
+		"""Test a complete filesystem workflow with JSON and CSV files."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			# Create filesystem
+			fs = FileSystem(base_dir=tmp_dir, create_default_files=True)
+
+			# Write JSON configuration file
+			config_json = '{"app": {"name": "TestApp", "version": "1.0"}, "database": {"host": "localhost", "port": 5432}}'
+			await fs.write_file('config.json', config_json)
+
+			# Write CSV data file
+			users_csv = 'id,name,email,age\n1,John Doe,john@example.com,30\n2,Jane Smith,jane@example.com,25'
+			await fs.write_file('users.csv', users_csv)
+
+			# Append more data to CSV
+			await fs.append_file('users.csv', '\n3,Bob Johnson,bob@example.com,35')
+
+			# Update JSON configuration
+			updated_config = '{"app": {"name": "TestApp", "version": "1.1"}, "database": {"host": "localhost", "port": 5432}, "features": {"logging": true}}'
+			await fs.write_file('config.json', updated_config)
+
+			# Create another JSON file for API responses
+			api_response = '{"status": "success", "data": [{"id": 1, "name": "Item 1"}, {"id": 2, "name": "Item 2"}]}'
+			await fs.write_file('api_response.json', api_response)
+
+			# Create a products CSV file
+			products_csv = (
+				'sku,name,price,category\nLAP001,Gaming Laptop,1299.99,Electronics\nMOU001,Wireless Mouse,29.99,Accessories'
+			)
+			await fs.write_file('products.csv', products_csv)
+
+			# Verify file listing
+			files = fs.list_files()
+			expected_files = ['todo.md', 'config.json', 'users.csv', 'api_response.json', 'products.csv']
+			assert len(files) == len(expected_files)
+			for expected_file in expected_files:
+				assert expected_file in files
+
+			# Verify JSON file contents
+			config_file = fs.get_file('config.json')
+			assert config_file is not None
+			assert isinstance(config_file, JsonFile)
+			assert config_file.content == updated_config
+
+			api_file = fs.get_file('api_response.json')
+			assert api_file is not None
+			assert isinstance(api_file, JsonFile)
+			assert api_file.content == api_response
+
+			# Verify CSV file contents
+			users_file = fs.get_file('users.csv')
+			assert users_file is not None
+			assert isinstance(users_file, CsvFile)
+			expected_users_content = 'id,name,email,age\n1,John Doe,john@example.com,30\n2,Jane Smith,jane@example.com,25\n3,Bob Johnson,bob@example.com,35'
+			assert users_file.content == expected_users_content
+
+			products_file = fs.get_file('products.csv')
+			assert products_file is not None
+			assert isinstance(products_file, CsvFile)
+			assert products_file.content == products_csv
+
+			# Test state persistence with JSON and CSV files
+			state = fs.get_state()
+			fs.nuke()
+
+			# Restore from state
+			fs2 = FileSystem.from_state(state)
+
+			# Verify restoration
+			assert len(fs2.files) == len(expected_files)
+
+			# Verify JSON files were restored correctly
+			restored_config = fs2.get_file('config.json')
+			assert restored_config is not None
+			assert isinstance(restored_config, JsonFile)
+			assert restored_config.content == updated_config
+
+			restored_api = fs2.get_file('api_response.json')
+			assert restored_api is not None
+			assert isinstance(restored_api, JsonFile)
+			assert restored_api.content == api_response
+
+			# Verify CSV files were restored correctly
+			restored_users = fs2.get_file('users.csv')
+			assert restored_users is not None
+			assert isinstance(restored_users, CsvFile)
+			assert restored_users.content == expected_users_content
+
+			restored_products = fs2.get_file('products.csv')
+			assert restored_products is not None
+			assert isinstance(restored_products, CsvFile)
+			assert restored_products.content == products_csv
+
+			# Verify files exist on disk
+			for filename in expected_files:
+				if filename != 'todo.md':  # Skip todo.md as it's already tested
+					assert (fs2.data_dir / filename).exists()
+
+			fs2.nuke()
+
+	async def test_from_state_with_json_csv_files(self, temp_filesystem):
+		"""Test restoring filesystem from state with JSON and CSV files."""
+		fs = temp_filesystem
+
+		# Add JSON and CSV content
+		await fs.write_file('data.json', '{"version": "1.0", "users": [{"name": "John", "age": 30}]}')
+		await fs.write_file('users.csv', 'name,age,city\nJohn,30,New York\nJane,25,London')
+		await fs.write_file('config.json', '{"debug": true, "port": 4242}')
+		await fs.write_file('products.csv', 'id,name,price\n1,Laptop,999.99\n2,Mouse,29.99')
+
+		# Get state
+		state = fs.get_state()
+
+		# Create new filesystem from state
+		fs2 = FileSystem.from_state(state)
+
+		# Verify restoration
+		assert fs2.base_dir == fs.base_dir
+		assert len(fs2.files) == len(fs.files)
+
+		# Verify JSON file contents
+		json_file = fs2.get_file('data.json')
+		assert json_file is not None
+		assert isinstance(json_file, JsonFile)
+		assert json_file.content == '{"version": "1.0", "users": [{"name": "John", "age": 30}]}'
+
+		config_file = fs2.get_file('config.json')
+		assert config_file is not None
+		assert isinstance(config_file, JsonFile)
+		assert config_file.content == '{"debug": true, "port": 4242}'
+
+		# Verify CSV file contents
+		csv_file = fs2.get_file('users.csv')
+		assert csv_file is not None
+		assert isinstance(csv_file, CsvFile)
+		assert csv_file.content == 'name,age,city\nJohn,30,New York\nJane,25,London'
+
+		products_file = fs2.get_file('products.csv')
+		assert products_file is not None
+		assert isinstance(products_file, CsvFile)
+		assert products_file.content == 'id,name,price\n1,Laptop,999.99\n2,Mouse,29.99'
+
+		# Verify files exist on disk
+		assert (fs2.data_dir / 'data.json').exists()
+		assert (fs2.data_dir / 'users.csv').exists()
+		assert (fs2.data_dir / 'config.json').exists()
+		assert (fs2.data_dir / 'products.csv').exists()
+
+		# Verify disk contents match
+		assert (fs2.data_dir / 'data.json').read_text() == '{"version": "1.0", "users": [{"name": "John", "age": 30}]}'
+		assert (fs2.data_dir / 'users.csv').read_text() == 'name,age,city\nJohn,30,New York\nJane,25,London'
+
+		# Clean up second filesystem
+		fs2.nuke()
+
+	def test_nuke(self, empty_filesystem):
+		"""Test filesystem destruction."""
+		fs = empty_filesystem
+
+		# Create a file to ensure directory has content
+		fs.data_dir.mkdir(exist_ok=True)
+		test_file = fs.data_dir / 'test.txt'
+		test_file.write_text('test')
+		assert test_file.exists()
+
+		# Nuke the filesystem
+		fs.nuke()
+
+		# Verify directory is removed
+		assert not fs.data_dir.exists()
+
+	def test_get_dir(self, temp_filesystem):
+		"""Test getting the filesystem directory."""
+		fs = temp_filesystem
+
+		directory = fs.get_dir()
+		assert directory == fs.data_dir
+		assert directory.exists()
+		assert directory.name == DEFAULT_FILE_SYSTEM_PATH
+
+
+class TestFileSystemEdgeCases:
+	"""Test edge cases and error handling."""
+
+	def test_filesystem_with_string_path(self):
+		"""Test FileSystem creation with string path."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			fs = FileSystem(base_dir=tmp_dir, create_default_files=False)
+			assert isinstance(fs.base_dir, Path)
+			assert fs.base_dir.exists()
+			fs.nuke()
+
+	def test_filesystem_with_path_object(self):
+		"""Test FileSystem creation with Path object."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			path_obj = Path(tmp_dir)
+			fs = FileSystem(base_dir=path_obj, create_default_files=False)
+			assert isinstance(fs.base_dir, Path)
+			assert fs.base_dir == path_obj
+			fs.nuke()
+
+	def test_filesystem_recreates_data_dir(self):
+		"""Test that FileSystem recreates data directory if it exists."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			# Create filesystem
+			fs1 = FileSystem(base_dir=tmp_dir, create_default_files=True)
+			data_dir = fs1.data_dir
+
+			# Add a custom file
+			custom_file = data_dir / 'custom.txt'
+			custom_file.write_text('custom content')
+			assert custom_file.exists()
+
+			# Create another filesystem with same base_dir (should clean data_dir)
+			fs2 = FileSystem(base_dir=tmp_dir, create_default_files=True)
+
+			# Custom file should be gone, default files should exist
+			assert not custom_file.exists()
+			assert (fs2.data_dir / 'todo.md').exists()
+
+			fs2.nuke()
+
+	async def test_write_file_exception_handling(self):
+		"""Test exception handling in write_file."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			fs = FileSystem(base_dir=tmp_dir, create_default_files=False)
+
+			# Test with invalid extension
+			result = await fs.write_file('test.invalid', 'content')
+			assert result == INVALID_FILENAME_ERROR_MESSAGE
+
+			fs.nuke()
+
+	def test_from_state_with_unknown_file_type(self):
+		"""Test restoring state with unknown file types (should skip them)."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			# Create a state with unknown file type
+			state = FileSystemState(
+				files={
+					'test.md': {'type': 'MarkdownFile', 'data': {'name': 'test', 'content': 'test content'}},
+					'unknown.txt': {'type': 'UnknownFileType', 'data': {'name': 'unknown', 'content': 'unknown content'}},
+				},
+				base_dir=tmp_dir,
+				extracted_content_count=0,
+			)
+
+			# Restore from state
+			fs = FileSystem.from_state(state)
+
+			# Should only have the known file type
+			assert 'test.md' in fs.files
+			assert 'unknown.txt' not in fs.files
+			assert len(fs.files) == 1
+
+			fs.nuke()
+
+
+class TestFileSystemIntegration:
+	"""Integration tests for FileSystem with real file operations."""
+
+	async def test_complete_workflow(self):
+		"""Test a complete filesystem workflow."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			# Create filesystem
+			fs = FileSystem(base_dir=tmp_dir, create_default_files=True)
+
+			# Write to results file
+			await fs.write_file('results.md', '# Test Results\n## Section 1\nInitial results.')
+
+			# Append more content
+			await fs.append_file('results.md', '\n## Section 2\nAdditional findings.')
+
+			# Create a notes file
+			await fs.write_file('notes.txt', 'Important notes:\n- Note 1\n- Note 2')
+
+			# Save extracted content
+			await fs.save_extracted_content('Extracted data from web page')
+			await fs.save_extracted_content('Second extraction')
+
+			# Verify file listing
+			files = fs.list_files()
+			assert len(files) == 5  # results.md, todo.md, notes.txt, 2 extracted files
+
+			# Verify content
+			file_obj = fs.get_file('results.md')
+			assert file_obj is not None
+			results_content = file_obj.content
+			assert '# Test Results' in results_content
+			assert '## Section 1' in results_content
+			assert '## Section 2' in results_content
+			assert 'Additional findings.' in results_content
+
+			# Test state persistence
+			state = fs.get_state()
+			fs.nuke()
+
+			# Restore from state
+			fs2 = FileSystem.from_state(state)
+
+			# Verify restoration
+			assert len(fs2.files) == 5
+			file_obj = fs2.get_file('results.md')
+			assert file_obj is not None
+			assert file_obj.content == results_content
+			file_obj = fs2.get_file('notes.txt')
+			assert file_obj is not None
+			assert file_obj.content == 'Important notes:\n- Note 1\n- Note 2'
+			assert fs2.extracted_content_count == 2
+
+			# Verify files exist on disk
+			for filename in files:
+				assert (fs2.data_dir / filename).exists()
+
+			fs2.nuke()
+
+	async def test_concurrent_operations(self):
+		"""Test concurrent file operations."""
+		with tempfile.TemporaryDirectory() as tmp_dir:
+			fs = FileSystem(base_dir=tmp_dir, create_default_files=False)
+
+			# Create multiple files concurrently
+			tasks = []
+			for i in range(5):
+				tasks.append(fs.write_file(f'file_{i}.md', f'Content for file {i}'))
+
+			# Wait for all operations to complete
+			results = await asyncio.gather(*tasks)
+
+			# Verify all operations succeeded
+			for result in results:
+				assert 'successfully' in result
+
+			# Verify all files were created
+			assert len(fs.files) == 5
+			for i in range(5):
+				assert f'file_{i}.md' in fs.files
+				file_obj = fs.get_file(f'file_{i}.md')
+				assert file_obj is not None
+				assert file_obj.content == f'Content for file {i}'
+
+			fs.nuke()

chapter9/browser-use-rpa/browser-use/tests/ci/test_llm_anthropic_502_error.py

@@ -0,0 +1,74 @@
+"""Test for handling Anthropic 502 errors"""
+
+import pytest
+from anthropic import APIStatusError
+
+from browser_use.llm.anthropic.chat import ChatAnthropic
+from browser_use.llm.exceptions import ModelProviderError
+from browser_use.llm.messages import BaseMessage, UserMessage
+
+
+@pytest.mark.asyncio
+async def test_anthropic_502_error_handling(monkeypatch):
+	"""Test that ChatAnthropic properly handles 502 errors from the API"""
+	# Create a ChatAnthropic instance
+	chat = ChatAnthropic(model='claude-3-5-sonnet-20240620', api_key='test-key')
+
+	# Create test messages
+	messages: list[BaseMessage] = [UserMessage(content='Test message')]
+
+	# Mock the client to raise a 502 error
+	class MockClient:
+		class Messages:
+			async def create(self, **kwargs):
+				# Simulate a 502 error from Anthropic API
+				import httpx
+
+				request = httpx.Request('POST', 'https://api.anthropic.com/v1/messages')
+				response = httpx.Response(status_code=502, headers={}, content=b'Bad Gateway', request=request)
+				raise APIStatusError(
+					message='Bad Gateway', response=response, body={'error': {'message': 'Bad Gateway', 'type': 'server_error'}}
+				)
+
+		messages = Messages()
+
+	# Replace the client with our mock
+	monkeypatch.setattr(chat, 'get_client', lambda: MockClient())
+
+	# Test that the error is properly caught and re-raised as ModelProviderError
+	with pytest.raises(ModelProviderError) as exc_info:
+		await chat.ainvoke(messages)
+
+	# Verify the error details
+	assert exc_info.value.args[0] == 'Bad Gateway'
+	assert exc_info.value.args[1] == 502
+	assert str(exc_info.value) == "('Bad Gateway', 502)"
+
+
+@pytest.mark.asyncio
+async def test_anthropic_error_does_not_access_usage(monkeypatch):
+	"""Test that error handling doesn't try to access usage attribute on error responses"""
+	chat = ChatAnthropic(model='claude-3-5-sonnet-20240620', api_key='test-key')
+
+	messages: list[BaseMessage] = [UserMessage(content='Test message')]
+
+	# Mock the client to return a string instead of a proper response
+	class MockClient:
+		class Messages:
+			async def create(self, **kwargs):
+				# This simulates what might happen if the API returns an unexpected response
+				# that gets parsed as a string
+				return 'Error: Bad Gateway'
+
+		messages = Messages()
+
+	monkeypatch.setattr(chat, 'get_client', lambda: MockClient())
+
+	# This should raise a ModelProviderError with a clear message
+	with pytest.raises(ModelProviderError) as exc_info:
+		await chat.ainvoke(messages)
+
+	# The error should be about unexpected response type, not missing 'usage' attribute
+	assert "'str' object has no attribute 'usage'" not in str(exc_info.value)
+	assert 'Unexpected response type from Anthropic API' in str(exc_info.value)
+	assert exc_info.value.args[1] == 502

chapter9/browser-use-rpa/browser-use/tests/ci/test_llm_custom_structured_ouput.py

@@ -0,0 +1,90 @@
+import json
+import os
+
+import tiktoken
+
+from browser_use.agent.views import AgentOutput
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.tools.service import Tools
+
+
+def test_optimized_schema():
+	"""Test the optimized schema generation and save to file."""
+
+	# Create tools and get all registered actions
+	tools = Tools()
+	ActionModel = tools.registry.create_action_model()
+
+	# Create the agent output model with custom actions
+	agent_output_model = AgentOutput.type_with_custom_actions(ActionModel)
+
+	# Get original schema for comparison
+	original_schema = agent_output_model.model_json_schema()
+
+	# Create the optimized schema
+	optimized_schema = SchemaOptimizer.create_optimized_json_schema(agent_output_model)
+
+	# Create tmp directory if it doesn't exist
+	os.makedirs('./tmp', exist_ok=True)
+
+	# Save optimized schema
+	with open('./tmp/optimized_schema.json', 'w') as f:
+		json.dump(optimized_schema, f, separators=(',', ':'), indent=2)
+
+	print('✅ Optimized schema generated and saved to ./tmp/optimized_schema.json')
+
+	# Compare token counts of both
+	try:
+		enc = tiktoken.encoding_for_model('gpt-4o')
+	except KeyError:
+		enc = tiktoken.get_encoding('cl100k_base')
+
+	original_tokens = len(enc.encode(json.dumps(original_schema)))
+	optimized_tokens = len(enc.encode(json.dumps(optimized_schema, separators=(',', ':'))))
+
+	savings = original_tokens - optimized_tokens
+	savings_percentage = (savings / original_tokens * 100) if original_tokens > 0 else 0
+
+	print('\n📊 Token Count Comparison:')
+	print(f'   Original schema: {original_tokens:,} tokens')
+	print(f'   Optimized schema: {optimized_tokens:,} tokens')
+	print(f'   Token savings: {savings:,} tokens ({savings_percentage:.1f}% reduction)')
+
+	# Count tokens per action in optimized schema
+	print('\n🔍 Tokens per Action in Optimized Schema:')
+
+	if 'properties' in optimized_schema and 'action' in optimized_schema['properties']:
+		action_prop = optimized_schema['properties']['action']
+		if 'items' in action_prop and 'anyOf' in action_prop['items']:
+			actions = action_prop['items']['anyOf']
+
+			total_action_tokens = 0
+			for i, action in enumerate(actions):
+				action_json = json.dumps(action, separators=(',', ':'))
+				action_tokens = len(enc.encode(action_json))
+				total_action_tokens += action_tokens
+
+				# Try to get action name from the schema
+				action_name = 'Unknown'
+				if 'properties' in action:
+					# Get the first property that's not common ones like 'index', 'reasoning'
+					for prop_name in action['properties'].keys():
+						if prop_name not in ['index', 'reasoning']:
+							action_name = prop_name
+							break
+
+				print(f'   Action {i + 1} ({action_name}): {action_tokens:,} tokens')
+
+			print('\n📈 Summary:')
+			print(f'   Total actions: {len(actions)}')
+			print(f'   Total action tokens: {total_action_tokens:,} tokens')
+			print(f'   Average tokens per action: {total_action_tokens // len(actions):,} tokens')
+			print(f'   Action tokens as % of total: {(total_action_tokens / optimized_tokens * 100):.1f}%')
+		else:
+			print('   No actions found in expected schema structure')
+	else:
+		print('   No action property found in optimized schema')
+
+
+if __name__ == '__main__':
+	test_optimized_schema()

chapter9/browser-use-rpa/browser-use/tests/ci/test_llm_gemini_type_field_fix.py

@@ -0,0 +1,107 @@
+"""
+Test to reproduce and verify fix for GitHub issue #2470:
+"Python field with name 'type' handled differently between Gemini and OpenAI GPT"
+"""
+
+from browser_use.llm.google.chat import ChatGoogle
+from browser_use.llm.schema import SchemaOptimizer
+
+
+class TestGeminiTypeFieldHandling:
+	"""Test class for reproducing the type field issue with Gemini schema processing."""
+
+	def test_gemini_schema_with_dict_type_field(self):
+		"""
+		Test that Gemini schema processing handles dict 'type' field gracefully.
+		Reproduces the AttributeError: 'dict' object has no attribute 'upper'
+		"""
+		chat_google = ChatGoogle(model='gemini-2.0-flash-exp')
+
+		# Schema with dict instead of string in type field
+		problematic_schema = {'type': {'malformed': 'dict_type'}, 'properties': {}}
+
+		result = chat_google._fix_gemini_schema(problematic_schema)
+		assert result is not None
+		assert isinstance(result, dict)
+		assert result['type'] == {'malformed': 'dict_type'}
+
+	def test_gemini_schema_with_nested_dict_type_field(self):
+		"""
+		Test that nested dict 'type' fields are handled gracefully.
+		"""
+		chat_google = ChatGoogle(model='gemini-2.0-flash-exp')
+
+		# Schema with nested dict type field
+		problematic_schema = {
+			'type': 'object',
+			'properties': {'nested_field': {'type': {'malformed': 'dict_instead_of_string'}, 'properties': {}}},
+		}
+
+		result = chat_google._fix_gemini_schema(problematic_schema)
+		assert result is not None
+		assert isinstance(result, dict)
+		nested_type = result['properties']['nested_field']['type']
+		assert nested_type == {'malformed': 'dict_instead_of_string'}
+
+	def test_gemini_schema_with_none_type_field(self):
+		"""Test handling of None type field."""
+		chat_google = ChatGoogle(model='gemini-2.0-flash-exp')
+
+		problematic_schema = {'type': 'object', 'properties': {'nested_field': {'type': None, 'properties': {}}}}
+
+		result = chat_google._fix_gemini_schema(problematic_schema)
+		assert result is not None
+
+	def test_gemini_schema_with_valid_string_type(self):
+		"""Test that valid string type fields work correctly."""
+		chat_google = ChatGoogle(model='gemini-2.0-flash-exp')
+
+		valid_schema = {'type': 'object', 'properties': {'nested_field': {'type': 'object', 'properties': {}}}}
+
+		# Should work without issues
+		result = chat_google._fix_gemini_schema(valid_schema)
+		assert result is not None
+		assert isinstance(result, dict)
+
+	def test_gemini_schema_with_empty_properties_object(self):
+		"""Test handling of empty properties in object type."""
+		chat_google = ChatGoogle(model='gemini-2.0-flash-exp')
+
+		schema_with_empty_props = {
+			'type': 'object',
+			'properties': {
+				'empty_object': {
+					'type': 'object',
+					'properties': {},  # Empty properties should get placeholder
+				}
+			},
+		}
+
+		result = chat_google._fix_gemini_schema(schema_with_empty_props)
+
+		nested_props = result['properties']['empty_object']['properties']
+		assert '_placeholder' in nested_props
+		assert nested_props['_placeholder']['type'] == 'string'
+
+	def test_consistency_between_providers(self):
+		"""
+		Test that both Gemini and OpenAI handle schemas consistently.
+		The original issue was that Gemini would fail where OpenAI succeeded.
+		"""
+		from pydantic import BaseModel, Field
+
+		# Create a test model that generates a schema with dict type
+		class TestModel(BaseModel):
+			field_with_dict_type: dict = Field(default_factory=dict)
+
+		# OpenAI uses SchemaOptimizer directly
+		openai_schema = SchemaOptimizer.create_optimized_json_schema(TestModel)
+		assert openai_schema is not None
+
+		# Gemini processes the schema through _fix_gemini_schema
+		chat_google = ChatGoogle(model='gemini-2.0-flash-exp')
+		gemini_result = chat_google._fix_gemini_schema(openai_schema)
+		assert gemini_result is not None
+
+		# Both should handle the schema without errors
+		# This demonstrates that the fix makes Gemini consistent with OpenAI

chapter9/browser-use-rpa/browser-use/tests/ci/test_llm_schema_optimizer.py

@@ -0,0 +1,66 @@
+"""
+Tests for the SchemaOptimizer to ensure it correctly processes and
+optimizes the schemas for agent actions without losing information.
+"""
+
+from pydantic import BaseModel
+
+from browser_use.agent.views import AgentOutput
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.tools.service import Tools
+
+
+class ProductInfo(BaseModel):
+	"""A sample structured output model with multiple fields."""
+
+	price: str
+	title: str
+	rating: float | None = None
+
+
+def test_optimizer_preserves_all_fields_in_structured_done_action():
+	"""
+	Ensures the SchemaOptimizer does not drop fields from a custom structured
+	output model when creating the schema for the 'done' action.
+
+	This test specifically checks for a bug where fields were being lost
+	during the optimization process.
+	"""
+	# 1. Setup a tools with a custom output model, simulating an Agent
+	#    being created with an `output_model_schema`.
+	tools = Tools(output_model=ProductInfo)
+
+	# 2. Get the dynamically created AgentOutput model, which includes all registered actions.
+	ActionModel = tools.registry.create_action_model()
+	agent_output_model = AgentOutput.type_with_custom_actions(ActionModel)
+
+	# 3. Run the schema optimizer on the agent's output model.
+	optimized_schema = SchemaOptimizer.create_optimized_json_schema(agent_output_model)
+
+	# 4. Find the 'done' action schema within the optimized output.
+	# The path is properties -> action -> items -> anyOf -> [schema with 'done'].
+	done_action_schema = None
+	actions_schemas = optimized_schema.get('properties', {}).get('action', {}).get('items', {}).get('anyOf', [])
+	for action_schema in actions_schemas:
+		if 'done' in action_schema.get('properties', {}):
+			done_action_schema = action_schema
+			break
+
+	# 5. Assert that the 'done' action schema was successfully found.
+	assert done_action_schema is not None, "Could not find 'done' action in the optimized schema."
+
+	# 6. Navigate to the schema for our custom data model within the 'done' action.
+	# The path is properties -> done -> properties -> data -> properties.
+	done_params_schema = done_action_schema.get('properties', {}).get('done', {})
+	structured_data_schema = done_params_schema.get('properties', {}).get('data', {})
+	final_properties = structured_data_schema.get('properties', {})
+
+	# 7. Assert that the set of fields in the optimized schema matches the original model's fields.
+	original_fields = set(ProductInfo.model_fields.keys())
+	optimized_fields = set(final_properties.keys())
+
+	assert original_fields == optimized_fields, (
+		f"Field mismatch between original and optimized structured 'done' action schema.\n"
+		f'Missing from optimized: {original_fields - optimized_fields}\n'
+		f'Unexpected in optimized: {optimized_fields - original_fields}'
+	)

chapter9/browser-use-rpa/browser-use/tests/ci/test_radio_buttons.html

@@ -0,0 +1,106 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Radio Button Test</title>
+</head>
+<body>
+    <h1>Radio Button Test Page</h1>
+    
+    <form>
+        <fieldset>
+            <legend>Select your favorite color:</legend>
+            
+            <label>
+                <input type="radio" name="color" value="red" id="radio-red">
+                Red
+            </label>
+            <br>
+            
+            <label>
+                <input type="radio" name="color" value="blue" id="radio-blue">
+                Blue
+            </label>
+            <br>
+            
+            <label>
+                <input type="radio" name="color" value="green" id="radio-green">
+                Green
+            </label>
+            <br>
+        </fieldset>
+        
+        <fieldset>
+            <legend>Select your favorite animal:</legend>
+            
+            <label>
+                <input type="radio" name="animal" value="cat" id="radio-cat">
+                Cat
+            </label>
+            <br>
+            
+            <label>
+                <input type="radio" name="animal" value="dog" id="radio-dog">
+                Dog
+            </label>
+            <br>
+            
+            <label>
+                <input type="radio" name="animal" value="bird" id="radio-bird">
+                Bird
+            </label>
+            <br>
+        </fieldset>
+        
+        <div id="result-message" style="margin-top: 20px; padding: 10px; background-color: #f0f0f0; display: none;">
+            <p id="secret-text"></p>
+        </div>
+    </form>
+    
+    <script>
+        function checkSelection() {
+            const colorRadios = document.querySelectorAll('input[name="color"]');
+            const animalRadios = document.querySelectorAll('input[name="animal"]');
+            
+            let selectedColor = null;
+            let selectedAnimal = null;
+            
+            // Get selected color
+            for (const radio of colorRadios) {
+                if (radio.checked) {
+                    selectedColor = radio.value;
+                    break;
+                }
+            }
+            
+            // Get selected animal
+            for (const radio of animalRadios) {
+                if (radio.checked) {
+                    selectedAnimal = radio.value;
+                    break;
+                }
+            }
+            
+            const resultDiv = document.getElementById('result-message');
+            const secretText = document.getElementById('secret-text');
+            
+            // Show secret if both Blue and Dog are selected
+            if (selectedColor === 'blue' && selectedAnimal === 'dog') {
+                secretText.textContent = 'SECRET_SUCCESS_12345: Blue dog combination unlocked!';
+                resultDiv.style.display = 'block';
+                resultDiv.style.backgroundColor = '#d4edda';
+            } else if (selectedColor && selectedAnimal) {
+                secretText.textContent = `Selected: ${selectedColor} ${selectedAnimal}`;
+                resultDiv.style.display = 'block';
+                resultDiv.style.backgroundColor = '#f8d7da';
+            } else {
+                resultDiv.style.display = 'none';
+            }
+        }
+        
+        // Add event listeners to all radio buttons
+        document.querySelectorAll('input[type="radio"]').forEach(radio => {
+            radio.addEventListener('change', checkSelection);
+        });
+    </script>
+</body>
+</html>

chapter9/browser-use-rpa/browser-use/tests/ci/test_radio_buttons.py

@@ -0,0 +1,105 @@
+# @file purpose: Test radio button interactions and serialization in browser-use
+"""
+Test file for verifying radio button clicking functionality and DOM serialization.
+
+This test creates a simple HTML page with radio buttons, sends an agent to click them,
+and logs the final agent message to show how radio buttons are represented in the serializer.
+
+The serialization shows radio buttons as:
+[index]<input type=radio name=groupname value=optionvalue checked=true/false />
+
+Usage:
+    uv run pytest tests/ci/test_radio_buttons.py -v -s
+
+Note: This test requires a real LLM API key and is skipped in CI environments.
+"""
+
+import os
+from pathlib import Path
+
+import pytest
+from pytest_httpserver import HTTPServer
+
+from browser_use.agent.service import Agent
+from browser_use.browser import BrowserSession
+from browser_use.browser.profile import BrowserProfile
+
+
+@pytest.fixture(scope='session')
+def http_server():
+	"""Create and provide a test HTTP server that serves static content."""
+	server = HTTPServer()
+	server.start()
+
+	# Read the HTML file content
+	html_file = Path(__file__).parent / 'test_radio_buttons.html'
+	with open(html_file) as f:
+		html_content = f.read()
+
+	# Add route for radio buttons test page
+	server.expect_request('/radio-test').respond_with_data(
+		html_content,
+		content_type='text/html',
+	)
+
+	yield server
+	server.stop()
+
+
+@pytest.fixture(scope='session')
+def base_url(http_server):
+	"""Return the base URL for the test HTTP server."""
+	return f'http://{http_server.host}:{http_server.port}'
+
+
+@pytest.fixture(scope='module')
+async def browser_session():
+	"""Create and provide a Browser instance with security disabled."""
+	browser_session = BrowserSession(
+		browser_profile=BrowserProfile(
+			headless=True,
+			user_data_dir=None,
+			keep_alive=True,
+		)
+	)
+	await browser_session.start()
+	yield browser_session
+	await browser_session.kill()
+
+
+@pytest.mark.skipif(
+	os.getenv('CI') == 'true' or os.getenv('GITHUB_ACTIONS') == 'true',
+	reason='Skipped in CI: requires real LLM API key which blocks other tests',
+)
+class TestRadioButtons:
+	"""Test cases for radio button interactions."""
+
+	async def test_radio_button_clicking(self, browser_session, base_url):
+		"""Test that agent can click radio buttons by checking for secret message."""
+
+		task = f"Go to {base_url}/radio-test and click on the 'Blue' radio button and the 'Dog' radio button. After clicking both buttons, look for any text message that appears on the page and report exactly what you see."
+
+		agent = Agent(
+			task=task,
+			browser_session=browser_session,
+			max_actions_per_step=5,
+			flash_mode=True,
+		)
+
+		# Run the agent
+		history = await agent.run(max_steps=8)
+
+		# Check if the secret message appears in the final response
+		secret_found = False
+		final_response = history.final_result()
+
+		if final_response and 'SECRET_SUCCESS_12345' in final_response:
+			secret_found = True
+			print('\n✅ SUCCESS: Secret message found! Radio buttons were clicked correctly.')
+
+		assert secret_found, (
+			"Secret message 'SECRET_SUCCESS_12345' should be present, indicating both Blue and Dog radio buttons were clicked. Actual response: "
+			+ str(final_response)
+		)
+
+		print(f'\n🎉 Test completed successfully! Agent completed {len(history)} steps and found the secret message.')

chapter9/browser-use-rpa/browser-use/tests/ci/test_registry_action_parameter_injection.py

@@ -0,0 +1,398 @@
+import asyncio
+import base64
+import socketserver
+
+import pytest
+from pytest_httpserver import HTTPServer
+
+from browser_use.browser import BrowserProfile, BrowserSession
+
+# Fix for httpserver hanging on shutdown - prevent blocking on socket close
+socketserver.ThreadingMixIn.block_on_close = False
+socketserver.ThreadingMixIn.daemon_threads = True
+
+
+class TestBrowserContext:
+	"""Tests for browser context functionality using real browser instances."""
+
+	@pytest.fixture(scope='session')
+	def http_server(self):
+		"""Create and provide a test HTTP server that serves static content."""
+		server = HTTPServer()
+		server.start()
+
+		# Add routes for test pages
+		server.expect_request('/').respond_with_data(
+			'<html><head><title>Test Home Page</title></head><body><h1>Test Home Page</h1><p>Welcome to the test site</p></body></html>',
+			content_type='text/html',
+		)
+
+		server.expect_request('/scroll_test').respond_with_data(
+			"""
+            <html>
+            <head>
+                <title>Scroll Test</title>
+                <style>
+                    body { height: 3000px; }
+                    .marker { position: absolute; }
+                    #top { top: 0; }
+                    #middle { top: 1000px; }
+                    #bottom { top: 2000px; }
+                </style>
+            </head>
+            <body>
+                <div id="top" class="marker">Top of the page</div>
+                <div id="middle" class="marker">Middle of the page</div>
+                <div id="bottom" class="marker">Bottom of the page</div>
+            </body>
+            </html>
+            """,
+			content_type='text/html',
+		)
+
+		yield server
+		server.stop()
+
+	@pytest.fixture(scope='session')
+	def base_url(self, http_server):
+		"""Return the base URL for the test HTTP server."""
+		return f'http://{http_server.host}:{http_server.port}'
+
+	@pytest.fixture(scope='module')
+	async def browser_session(self):
+		"""Create and provide a BrowserSession instance with security disabled."""
+		browser_session = BrowserSession(
+			browser_profile=BrowserProfile(
+				headless=True,
+				user_data_dir=None,
+				keep_alive=True,
+			)
+		)
+		await browser_session.start()
+		yield browser_session
+		await browser_session.kill()
+		# Ensure event bus is properly stopped
+		await browser_session.event_bus.stop(clear=True, timeout=5)
+
+	@pytest.mark.skip(reason='TODO: fix')
+	def test_is_url_allowed(self):
+		"""
+		Test the _is_url_allowed method to verify that it correctly checks URLs against
+		the allowed domains configuration.
+		"""
+		# Scenario 1: allowed_domains is None, any URL should be allowed.
+		from bubus import EventBus
+
+		from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+		config1 = BrowserProfile(allowed_domains=None, headless=True, user_data_dir=None)
+		context1 = BrowserSession(browser_profile=config1)
+		event_bus1 = EventBus()
+		watchdog1 = SecurityWatchdog(browser_session=context1, event_bus=event_bus1)
+		assert watchdog1._is_url_allowed('http://anydomain.com') is True
+		assert watchdog1._is_url_allowed('https://anotherdomain.org/path') is True
+
+		# Scenario 2: allowed_domains is provided.
+		# Note: match_url_with_domain_pattern defaults to https:// scheme when none is specified
+		allowed = ['https://example.com', 'http://example.com', 'http://*.mysite.org', 'https://*.mysite.org']
+		config2 = BrowserProfile(allowed_domains=allowed, headless=True, user_data_dir=None)
+		context2 = BrowserSession(browser_profile=config2)
+		event_bus2 = EventBus()
+		watchdog2 = SecurityWatchdog(browser_session=context2, event_bus=event_bus2)
+
+		# URL exactly matching
+		assert watchdog2._is_url_allowed('http://example.com') is True
+		# URL with subdomain (should not be allowed)
+		assert watchdog2._is_url_allowed('http://sub.example.com/path') is False
+		# URL with subdomain for wildcard pattern (should be allowed)
+		assert watchdog2._is_url_allowed('http://sub.mysite.org') is True
+		# URL that matches second allowed domain
+		assert watchdog2._is_url_allowed('https://mysite.org/page') is True
+		# URL with port number, still allowed (port is stripped)
+		assert watchdog2._is_url_allowed('http://example.com:4242') is True
+		assert watchdog2._is_url_allowed('https://example.com:443') is True
+
+		# Scenario 3: Malformed URL or empty domain
+		# urlparse will return an empty netloc for some malformed URLs.
+		assert watchdog2._is_url_allowed('notaurl') is False
+
+	def test_convert_simple_xpath_to_css_selector(self):
+		"""
+		Test removed: _convert_simple_xpath_to_css_selector method no longer exists.
+		"""
+		pass  # Method was removed from BrowserSession
+
+	def test_enhanced_css_selector_for_element(self):
+		"""
+		Test removed: _enhanced_css_selector_for_element method no longer exists.
+		"""
+		pass  # Method was removed from BrowserSession
+
+	@pytest.mark.asyncio
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_navigate_and_get_current_page(self, browser_session, base_url):
+		"""Test that navigate method changes the URL and get_current_page returns the proper page."""
+		# Navigate to the test page
+		from browser_use.browser.events import NavigateToUrlEvent
+
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/'))
+		await event
+
+		# Get the current page
+		url = await browser_session.get_current_page_url()
+
+		# Verify the page URL matches what we navigated to
+		assert f'{base_url}/' in url
+
+		# Verify the page title
+		title = await browser_session.get_current_page_title()
+		assert title == 'Test Home Page'
+
+	@pytest.mark.asyncio
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_refresh_page(self, browser_session, base_url):
+		"""Test that refresh_page correctly reloads the current page."""
+		# Navigate to the test page
+		from browser_use.browser.events import NavigateToUrlEvent
+
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/'))
+		await event
+
+		# Get the current page info before refresh
+		url_before = await browser_session.get_current_page_url()
+		title_before = await browser_session.get_current_page_title()
+
+		# Refresh the page
+		await browser_session.refresh()
+
+		# Get the current page info after refresh
+		url_after = await browser_session.get_current_page_url()
+		title_after = await browser_session.get_current_page_title()
+
+		# Verify it's still on the same URL
+		assert url_after == url_before
+
+		# Verify the page title is still correct
+		assert title_after == 'Test Home Page'
+
+	@pytest.mark.asyncio
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_execute_javascript(self, browser_session, base_url):
+		"""Test that execute_javascript correctly executes JavaScript in the current page."""
+		# Navigate to a test page
+		from browser_use.browser.events import NavigateToUrlEvent
+
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/'))
+		await event
+
+		# Execute a simple JavaScript snippet that returns a value
+		result = await browser_session.execute_javascript('document.title')
+
+		# Verify the result
+		assert result == 'Test Home Page'
+
+		# Execute JavaScript that modifies the page
+		await browser_session.execute_javascript("document.body.style.backgroundColor = 'red'")
+
+		# Verify the change by reading back the value
+		bg_color = await browser_session.execute_javascript('document.body.style.backgroundColor')
+		assert bg_color == 'red'
+
+	@pytest.mark.asyncio
+	@pytest.mark.skip(reason='TODO: fix')
+	@pytest.mark.skip(reason='get_scroll_info API changed - depends on page object that no longer exists')
+	async def test_get_scroll_info(self, browser_session, base_url):
+		"""Test that get_scroll_info returns the correct scroll position information."""
+		# Navigate to the scroll test page
+		from browser_use.browser.events import NavigateToUrlEvent
+
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/scroll_test'))
+		await event
+		page = await browser_session.get_current_page()
+
+		# Get initial scroll info
+		pixels_above_initial, pixels_below_initial = await browser_session.get_scroll_info(page)
+
+		# Verify initial scroll position
+		assert pixels_above_initial == 0, 'Initial scroll position should be at the top'
+		assert pixels_below_initial > 0, 'There should be content below the viewport'
+
+		# Scroll down the page
+		await browser_session.execute_javascript('window.scrollBy(0, 500)')
+		await asyncio.sleep(0.2)  # Brief delay for scroll to complete
+
+		# Get new scroll info
+		pixels_above_after_scroll, pixels_below_after_scroll = await browser_session.get_scroll_info(page)
+
+		# Verify new scroll position
+		assert pixels_above_after_scroll > 0, 'Page should be scrolled down'
+		assert pixels_above_after_scroll >= 400, 'Page should be scrolled down at least 400px'
+		assert pixels_below_after_scroll < pixels_below_initial, 'Less content should be below viewport after scrolling'
+
+	@pytest.mark.asyncio
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_take_screenshot(self, browser_session, base_url):
+		"""Test that take_screenshot returns a valid base64 encoded image."""
+		# Navigate to the test page
+		from browser_use.browser.events import NavigateToUrlEvent
+
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/'))
+		await event
+
+		# Take a screenshot
+		screenshot_base64 = await browser_session.take_screenshot()
+
+		# Verify the screenshot is a valid base64 string
+		assert isinstance(screenshot_base64, str)
+		assert len(screenshot_base64) > 0
+
+		# Verify it can be decoded as base64
+		try:
+			image_data = base64.b64decode(screenshot_base64)
+			# Verify the data starts with a valid image signature (PNG file header)
+			assert image_data[:8] == b'\x89PNG\r\n\x1a\n', 'Screenshot is not a valid PNG image'
+		except Exception as e:
+			pytest.fail(f'Failed to decode screenshot as base64: {e}')
+
+	@pytest.mark.asyncio
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_switch_tab_operations(self, browser_session, base_url):
+		"""Test tab creation, switching, and closing operations."""
+		# Navigate to home page in first tab
+		from browser_use.browser.events import NavigateToUrlEvent
+
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/'))
+		await event
+
+		# Create a new tab
+		await browser_session.create_new_tab(f'{base_url}/scroll_test')
+
+		# Verify we have two tabs now
+		tabs_info = await browser_session.get_tabs()
+		assert len(tabs_info) == 2, 'Should have two tabs open'
+
+		# Verify current tab is the scroll test page
+		current_url = await browser_session.get_current_page_url()
+		assert f'{base_url}/scroll_test' in current_url
+
+		# Switch back to the first tab
+		await browser_session.switch_to_tab(0)
+
+		# Verify we're back on the home page
+		current_url = await browser_session.get_current_page_url()
+		assert f'{base_url}/' in current_url
+
+		# Close the second tab
+		await browser_session.close_tab(1)
+
+		# Verify we have the expected number of tabs
+		# The first tab remains plus any about:blank tabs created by AboutBlankWatchdog
+		tabs_info = await browser_session.get_tabs_info()
+		# Filter out about:blank tabs created by the watchdog
+		non_blank_tabs = [tab for tab in tabs_info if 'about:blank' not in tab.url]
+		assert len(non_blank_tabs) == 1, (
+			f'Should have one non-blank tab open after closing the second, but got {len(non_blank_tabs)}: {non_blank_tabs}'
+		)
+		assert base_url in non_blank_tabs[0].url, 'The remaining tab should be the home page'
+
+	# TODO: highlighting doesn't exist anymore
+	# @pytest.mark.asyncio
+	# async def test_remove_highlights(self, browser_session, base_url):
+	# 	"""Test that remove_highlights successfully removes highlight elements."""
+	# 	# Navigate to a test page
+	# 	from browser_use.browser.events import NavigateToUrlEvent; event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/')
+
+	# 	# Add a highlight via JavaScript
+	# 	await browser_session.execute_javascript("""
+	#         const container = document.createElement('div');
+	#         container.id = 'playwright-highlight-container';
+	#         document.body.appendChild(container);
+
+	#         const highlight = document.createElement('div');
+	#         highlight.id = 'playwright-highlight-1';
+	#         container.appendChild(highlight);
+
+	#         const element = document.querySelector('h1');
+	#         element.setAttribute('browser-user-highlight-id', 'playwright-highlight-1');
+	#     """)
+
+	# 	# Verify the highlight container exists
+	# 	container_exists = await browser_session.execute_javascript(
+	# 		"document.getElementById('playwright-highlight-container') !== null"
+	# 	)
+	# 	assert container_exists, 'Highlight container should exist before removal'
+
+	# 	# Call remove_highlights
+	# 	await browser_session.remove_highlights()
+
+	# 	# Verify the highlight container was removed
+	# 	container_exists_after = await browser_session.execute_javascript(
+	# 		"document.getElementById('playwright-highlight-container') !== null"
+	# 	)
+	# 	assert not container_exists_after, 'Highlight container should be removed'
+
+	# 	# Verify the highlight attribute was removed from the element
+	# 	attribute_exists = await browser_session.execute_javascript(
+	# 		"document.querySelector('h1').hasAttribute('browser-user-highlight-id')"
+	# 	)
+	# 	assert not attribute_exists, 'browser-user-highlight-id attribute should be removed'
+
+	@pytest.mark.asyncio
+	@pytest.mark.skip(reason='TODO: fix')
+	async def test_custom_action_with_no_arguments(self, browser_session, base_url):
+		"""Test that custom actions with no arguments are handled correctly"""
+		from browser_use.agent.views import ActionResult
+		from browser_use.tools.registry.service import Registry
+
+		# Create a registry
+		registry = Registry()
+
+		# Register a custom action with no arguments
+		@registry.action('Some custom action with no args')
+		def simple_action():
+			return ActionResult(extracted_content='return some result')
+
+		# Navigate to a test page
+		from browser_use.browser.events import NavigateToUrlEvent
+
+		event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=f'{base_url}/'))
+		await event
+
+		# Execute the action
+		result = await registry.execute_action('simple_action', {})
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content == 'return some result'
+
+		# Test that the action model is created correctly
+		action_model = registry.create_action_model()
+
+		# The action should be in the model fields
+		assert 'simple_action' in action_model.model_fields
+
+		# Create an instance with the simple_action
+		action_instance = action_model(simple_action={})  # type: ignore[call-arg]
+
+		# Test that model_dump works correctly
+		dumped = action_instance.model_dump(exclude_unset=True)
+		assert 'simple_action' in dumped
+		assert dumped['simple_action'] == {}
+
+		# Test async version as well
+		@registry.action('Async custom action with no args')
+		async def async_simple_action():
+			return ActionResult(extracted_content='async result')
+
+		result = await registry.execute_action('async_simple_action', {})
+		assert result.extracted_content == 'async result'
+
+		# Test with special parameters but no regular arguments
+		@registry.action('Action with only special params')
+		async def special_params_only(browser_session):
+			current_url = await browser_session.get_current_page_url()
+			return ActionResult(extracted_content=f'Page URL: {current_url}')
+
+		result = await registry.execute_action('special_params_only', {}, browser_session=browser_session)
+		assert 'Page URL:' in result.extracted_content
+		assert base_url in result.extracted_content

chapter9/browser-use-rpa/browser-use/tests/ci/test_sync_client_auth.py

@@ -0,0 +1,993 @@
+"""
+Tests for OAuth2 device flow and cloud sync functionality.
+"""
+
+import json
+import tempfile
+from datetime import datetime
+from pathlib import Path
+
+import anyio
+import httpx
+import pytest
+from dotenv import load_dotenv
+from pytest_httpserver import HTTPServer
+
+# Load environment variables before any imports
+load_dotenv()
+
+
+from browser_use.agent.cloud_events import CreateAgentSessionEvent, CreateAgentTaskEvent
+from browser_use.sync.auth import TEMP_USER_ID, DeviceAuthClient
+from browser_use.sync.service import CloudSync
+
+# Define config dir for tests - not needed anymore since we'll use env vars
+
+
+@pytest.fixture
+def temp_config_dir(monkeypatch):
+	"""Create temporary config directory."""
+	with tempfile.TemporaryDirectory() as tmpdir:
+		temp_dir = Path(tmpdir) / '.config' / 'browseruse'
+		temp_dir.mkdir(parents=True, exist_ok=True)
+
+		# Use monkeypatch to set the environment variable
+		monkeypatch.setenv('BROWSER_USE_CONFIG_DIR', str(temp_dir))
+
+		yield temp_dir
+
+
+@pytest.fixture
+async def http_client(httpserver: HTTPServer):
+	"""Create a real HTTP client pointed at the test server"""
+	async with httpx.AsyncClient(base_url=httpserver.url_for('')) as client:
+		yield client
+
+
+class TestDeviceAuthClient:
+	"""Test DeviceAuthClient class."""
+
+	async def test_init_creates_config_dir(self, temp_config_dir, httpserver):
+		"""Test that initialization creates config directory."""
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+		assert temp_config_dir.exists()
+		assert (temp_config_dir / 'cloud_auth.json').exists() is False
+
+	async def test_load_credentials_no_file(self, temp_config_dir, httpserver):
+		"""Test loading credentials when file doesn't exist."""
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+		# When no file exists, auth_config should have no token/user_id
+		assert auth.auth_config.api_token is None
+		assert auth.auth_config.user_id is None
+		assert not auth.is_authenticated
+
+	async def test_save_and_load_credentials(self, temp_config_dir, httpserver):
+		"""Test saving and loading credentials."""
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+
+		# Update auth config and save
+		auth.auth_config.api_token = 'test-key-123'
+		auth.auth_config.user_id = 'test-user-123'
+		auth.auth_config.authorized_at = datetime.utcnow()
+		auth.auth_config.save_to_file()
+
+		# Load in a new instance
+		auth2 = DeviceAuthClient(base_url=httpserver.url_for(''))
+		assert auth2.auth_config.api_token == 'test-key-123'
+		assert auth2.auth_config.user_id == 'test-user-123'
+		assert auth2.is_authenticated
+		assert (temp_config_dir / 'cloud_auth.json').exists()
+
+		# Check file permissions (should be readable only by owner)
+		stat = (temp_config_dir / 'cloud_auth.json').stat()
+		assert oct(stat.st_mode)[-3:] == '600'
+
+	async def test_is_authenticated(self, temp_config_dir, httpserver):
+		"""Test authentication status check."""
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+
+		# Not authenticated initially
+		assert auth.is_authenticated is False
+
+		# Save credentials
+		auth.auth_config.api_token = 'test-key'
+		auth.auth_config.user_id = 'test-user'
+		auth.auth_config.save_to_file()
+
+		# Reload to verify persistence
+		auth2 = DeviceAuthClient(base_url=httpserver.url_for(''))
+		assert auth2.is_authenticated is True
+
+	async def test_get_credentials(self, temp_config_dir, httpserver):
+		"""Test getting credentials."""
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+
+		# No credentials initially
+		assert auth.api_token is None
+		assert auth.user_id == TEMP_USER_ID  # Should return temp user ID when not authenticated
+
+		# Save credentials
+		auth.auth_config.api_token = 'test-key'
+		auth.auth_config.user_id = 'test-user'
+
+		# Get credentials
+		assert auth.api_token == 'test-key'
+		assert auth.user_id == 'test-user'
+
+	async def test_start_device_flow(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Test starting device flow."""
+		# Set up the test server response
+		httpserver.expect_request(
+			'/api/v1/oauth/device/authorize',
+			method='POST',
+		).respond_with_json(
+			{
+				'device_code': 'test-device-code',
+				'user_code': 'ABCD-1234',
+				'verification_uri': 'https://example.com/device',
+				'verification_uri_complete': 'https://example.com/device?user_code=ABCD-1234',
+				'expires_in': 1800,
+				'interval': 5,
+			}
+		)
+
+		# Create auth client with injected http client
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+		result = await auth.start_device_authorization('test-session-id')
+
+		assert result['device_code'] == 'test-device-code'
+		assert result['user_code'] == 'ABCD-1234'
+		assert 'verification_uri' in result
+
+		# Verify the request was made correctly
+		request = httpserver.log[0][0]
+		assert request.method == 'POST'
+		# Get the body as string
+		body = request.get_data(as_text=True)
+		assert 'client_id=library' in body
+		assert 'agent_session_id=test-session-id' in body
+		assert 'device_id=' in body  # Should include device_id
+
+	async def test_poll_for_token_pending(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Test polling when authorization is pending."""
+		# Set up the test server to always return pending
+		httpserver.expect_request(
+			'/api/v1/oauth/device/token',
+			method='POST',
+		).respond_with_json(
+			{
+				'error': 'authorization_pending',
+				'error_description': 'Authorization pending',
+			}
+		)
+
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+		# Use very short timeout to avoid long test
+		result = await auth.poll_for_token('test-device-code', interval=0.1, timeout=0.5)
+
+		assert result is None
+		assert not auth.is_authenticated
+
+	async def test_poll_for_token_success(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Test successful token polling."""
+		# Set up the test server to return success immediately
+		httpserver.expect_request(
+			'/api/v1/oauth/device/token',
+			method='POST',
+		).respond_with_json(
+			{
+				'access_token': 'test-api-key',
+				'token_type': 'Bearer',
+				'user_id': 'test-user-123',
+				'scope': 'read write',
+			}
+		)
+
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+		result = await auth.poll_for_token('test-device-code')
+
+		assert result is not None
+		assert result['access_token'] == 'test-api-key'
+		assert result['user_id'] == 'test-user-123'
+
+	async def test_wait_for_authorization(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Test waiting for authorization with polling."""
+		# Track number of requests
+		request_count = 0
+
+		def handle_token_request(request):
+			nonlocal request_count
+			request_count += 1
+
+			from werkzeug.wrappers import Response
+
+			if request_count < 3:
+				# First two requests return pending
+				return Response(
+					json.dumps({'error': 'authorization_pending', 'error_description': 'Authorization pending'}),
+					status=200,
+					mimetype='application/json',
+				)
+			else:
+				# Third request returns success
+				return Response(
+					json.dumps(
+						{
+							'access_token': 'test-api-key',
+							'token_type': 'Bearer',
+							'user_id': 'test-user-123',
+							'scope': 'read write',
+						}
+					),
+					status=200,
+					mimetype='application/json',
+				)
+
+		# Set up auth endpoint
+		httpserver.expect_request(
+			'/api/v1/oauth/device/authorize',
+			method='POST',
+		).respond_with_json(
+			{
+				'device_code': 'test-device-code',
+				'user_code': 'ABCD-1234',
+				'verification_uri': 'https://example.com/device',
+				'verification_uri_complete': 'https://example.com/device?user_code=ABCD-1234',
+				'expires_in': 1800,
+				'interval': 0.1,  # Short interval for testing
+			}
+		)
+
+		# Set up token endpoint with custom handler
+		httpserver.expect_request(
+			'/api/v1/oauth/device/token',
+			method='POST',
+		).respond_with_handler(handle_token_request)
+
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+		success = await auth.authenticate(agent_session_id='test-session-id', show_instructions=False)
+
+		assert success is True
+		assert auth.is_authenticated
+		assert auth.api_token == 'test-api-key'
+		assert auth.user_id == 'test-user-123'
+		assert request_count == 3  # Verify it took 3 polls
+
+	async def test_wait_for_authorization_timeout(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Test timeout during authorization waiting."""
+		# Set up auth endpoint
+		httpserver.expect_request(
+			'/api/v1/oauth/device/authorize',
+			method='POST',
+		).respond_with_json(
+			{
+				'device_code': 'test-device-code',
+				'user_code': 'ABCD-1234',
+				'verification_uri': 'https://example.com/device',
+				'verification_uri_complete': 'https://example.com/device?user_code=ABCD-1234',
+				'expires_in': 1800,
+				'interval': 0.1,
+			}
+		)
+
+		# Set up token endpoint to always return pending
+		httpserver.expect_request(
+			'/api/v1/oauth/device/token',
+			method='POST',
+		).respond_with_json(
+			{
+				'error': 'authorization_pending',
+				'error_description': 'Authorization pending',
+			}
+		)
+
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+
+		# Call poll_for_token directly with short timeout
+		result = await auth.poll_for_token('test-device-code', interval=0.1, timeout=0.5)
+		assert result is None  # Should timeout and return None
+		assert not auth.is_authenticated
+
+	async def test_logout(self, temp_config_dir, httpserver):
+		"""Test logout functionality."""
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+
+		# Save credentials directly using auth_config
+		auth.auth_config.api_token = 'test-key'
+		auth.auth_config.user_id = 'test-user'
+		auth.auth_config.save_to_file()
+
+		assert auth.is_authenticated is True
+		assert (temp_config_dir / 'cloud_auth.json').exists()
+
+		# Clear auth (logout)
+		auth.clear_auth()
+
+		assert auth.is_authenticated is False
+		# Note: clear_auth() deletes the config file entirely for security
+		assert not (temp_config_dir / 'cloud_auth.json').exists()
+
+		# Verify a new client loads empty credentials when no file exists
+		auth2 = DeviceAuthClient(base_url=httpserver.url_for(''))
+		assert auth2.auth_config.api_token is None
+		assert auth2.auth_config.user_id is None
+
+
+class TestCloudSync:
+	"""Test CloudSync class."""
+
+	async def test_init(self, temp_config_dir, httpserver):
+		"""Test CloudSync initialization."""
+		service = CloudSync(base_url=httpserver.url_for(''))
+
+		assert service.base_url == httpserver.url_for('')
+		assert service.auth_client is not None
+		assert isinstance(service.auth_client, DeviceAuthClient)
+
+	async def test_send_event_authenticated(self, httpserver: HTTPServer, temp_config_dir):
+		"""Test sending event when authenticated."""
+		requests = []
+
+		def capture_request(request):
+			requests.append(
+				{
+					'headers': dict(request.headers),
+					'json': request.get_json(),
+				}
+			)
+			from werkzeug.wrappers import Response
+
+			return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
+
+		# Create authenticated service
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+		auth.auth_config.api_token = 'test-api-key'
+		auth.auth_config.user_id = 'test-user-123'
+
+		service = CloudSync(base_url=httpserver.url_for(''))
+		service.auth_client = auth
+		service.session_id = 'test-session-id'
+
+		# Send event
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Test task',
+				user_id='test-user-123',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+				device_id='test-device-id',
+			)
+		)
+
+		# Check request was made
+		assert len(requests) == 1
+		request_data = requests[0]
+
+		# Check auth header
+		assert request_data['headers']['Authorization'] == 'Bearer test-api-key'
+
+		# Check event data
+		json_data = request_data['json']
+		assert len(json_data['events']) == 1
+		event = json_data['events'][0]
+		assert event['event_type'] == 'CreateAgentTaskEvent'
+		assert event['user_id'] == 'test-user-123'
+		assert event['task'] == 'Test task'
+
+	async def test_send_event_pre_auth(self, httpserver: HTTPServer, temp_config_dir):
+		"""Test that non-session events are not sent when auth is not in progress."""
+		requests = []
+
+		def capture_request(request):
+			requests.append(
+				{
+					'headers': dict(request.headers),
+					'json': request.get_json(),
+				}
+			)
+			from werkzeug.wrappers import Response
+
+			return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
+
+		# Create unauthenticated service WITHOUT triggering auth
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+		# Don't set api_token - leave it unauthenticated
+
+		service = CloudSync(base_url=httpserver.url_for(''))
+		service.auth_client = auth
+		service.session_id = 'test-session-id'  # Set manually, don't trigger CreateAgentSessionEvent
+
+		# Send task event when NO auth is in progress (should be skipped)
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Test task',
+				user_id=TEMP_USER_ID,
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+				device_id='test-device-id',
+			)
+		)
+
+		# Check that no requests were made
+		assert len(requests) == 0
+
+	async def test_block_events_during_auth_progress(self, httpserver: HTTPServer, temp_config_dir):
+		"""Test that task events are BLOCKED when authentication is in progress (prevents data leak)."""
+		requests = []
+
+		def capture_request(request):
+			requests.append(
+				{
+					'headers': dict(request.headers),
+					'json': request.get_json(),
+				}
+			)
+			from werkzeug.wrappers import Response
+
+			return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
+
+		# Set up auth endpoints to simulate background auth in progress
+		httpserver.expect_request(
+			'/api/v1/oauth/device/authorize',
+			method='POST',
+		).respond_with_json(
+			{
+				'device_code': 'test-device-code',
+				'user_code': 'ABCD-1234',
+				'verification_uri': 'https://example.com/device',
+				'verification_uri_complete': 'https://example.com/device?user_code=ABCD-1234',
+				'expires_in': 1800,
+				'interval': 5,
+			}
+		)
+
+		httpserver.expect_request(
+			'/api/v1/oauth/device/token',
+			method='POST',
+		).respond_with_json(
+			{
+				'error': 'authorization_pending',
+				'error_description': 'Authorization pending',
+			}
+		)
+
+		# Create unauthenticated service
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+		# Don't set api_token - leave it unauthenticated
+
+		service = CloudSync(base_url=httpserver.url_for(''))
+		service.auth_client = auth
+
+		# Manually start an auth task to simulate the scenario where auth is in progress
+		import asyncio
+
+		async def fake_auth():
+			await asyncio.sleep(1)  # Simulate auth taking some time
+
+		service.auth_task = asyncio.create_task(fake_auth())
+
+		# Set session ID
+		service.session_id = 'test-session-id'
+
+		# Send task event while auth is in progress (should be BLOCKED for security)
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Test task during auth',
+				user_id=TEMP_USER_ID,
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+				device_id='test-device-id',
+			)
+		)
+
+		# Check that the task event was NOT sent (blocked for security during auth)
+		assert len(requests) == 0
+
+		# Clean up the background task to avoid test flakiness
+		if service.auth_task and not service.auth_task.done():
+			service.auth_task.cancel()
+			try:
+				await service.auth_task
+			except asyncio.CancelledError:
+				pass
+
+	async def test_authenticate_then_send(self, httpserver: HTTPServer, temp_config_dir):
+		"""Test that events are only sent after authentication."""
+		requests = []
+
+		def capture_request(request):
+			requests.append(
+				{
+					'headers': dict(request.headers),
+					'json': request.get_json(),
+				}
+			)
+			from werkzeug.wrappers import Response
+
+			return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
+
+		# Create service with unauthenticated auth client
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+		# Start unauthenticated
+
+		service = CloudSync(base_url=httpserver.url_for(''))
+		service.auth_client = auth
+		service.session_id = 'test-session-id'
+
+		# Send pre-auth event (should be skipped)
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Pre-auth task',
+				user_id=TEMP_USER_ID,
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+				device_id='test-device-id',
+			)
+		)
+
+		# No requests should have been made yet
+		assert len(requests) == 0
+
+		# Now authenticate the auth client
+		auth.auth_config.api_token = 'test-api-key'
+		auth.auth_config.user_id = 'test-user-123'
+
+		# Send post-auth event (should be sent)
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Post-auth task',
+				user_id='test-user-123',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+				device_id='test-device-id',
+			)
+		)
+
+		# Now exactly one request should have been made (the post-auth event)
+		assert len(requests) == 1
+		assert requests[0]['headers']['Authorization'] == 'Bearer test-api-key'
+		assert requests[0]['json']['events'][0]['user_id'] == 'test-user-123'
+		assert requests[0]['json']['events'][0]['task'] == 'Post-auth task'
+
+	async def test_error_handling(self, httpserver: HTTPServer, temp_config_dir):
+		"""Test error handling during event sending."""
+		# Set up server to return 500 error
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_data('Internal Server Error', status=500)
+
+		# Create service with real auth
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+		auth.auth_config.api_token = 'test-api-key'
+		auth.auth_config.user_id = 'test-user-123'
+
+		service = CloudSync(base_url=httpserver.url_for(''))
+		service.auth_client = auth
+		service.session_id = 'test-session-id'
+
+		# Send event - should not raise exception but handle gracefully
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Test task',
+				user_id='test-user-123',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+				device_id='test-device-id',
+			)
+		)
+
+		# Should handle error gracefully without crashing
+
+	# async def test_update_wal_events(self, temp_config_dir):
+	# 	"""Test updating WAL events with real user ID."""
+	# 	# Create real auth client
+	# 	auth = DeviceAuthClient(base_url='http://localhost:8000')
+	# 	auth.auth_config.api_token = 'test-api-key'
+	# 	auth.auth_config.user_id = 'test-user-123'
+
+	# 	service = CloudSync(
+	# 		base_url='http://localhost:8000'
+	# 	)
+	# 	service.auth_client = auth
+	# 	service.session_id = 'test-session-id'
+
+	# 	# Create the events directory structure that the method expects
+	# 	events_dir = temp_config_dir / 'events'
+	# 	events_dir.mkdir(exist_ok=True)
+
+	# 	# Create WAL file with temp user IDs
+	# 	wal_path = events_dir / f'{service.session_id}.jsonl'
+	# 	events = [
+	# 		{
+	# 			'event_type': 'CreateAgentTaskEvent',
+	# 			'user_id': '99999999-9999-9999-9999-999999999999',  # TEMP_USER_ID
+	# 			'task': 'Task 1',
+	# 		},
+	# 		{
+	# 			'event_type': 'UpdateAgentTaskEvent',
+	# 			'user_id': '99999999-9999-9999-9999-999999999999',  # TEMP_USER_ID
+	# 			'status': 'done',
+	# 		},
+	# 		{
+	# 			'event_type': 'CreateAgentStepEvent',
+	# 			'user_id': 'some-other-user',  # Different user, should still be updated
+	# 			'step': 1,
+	# 		},
+	# 	]
+
+	# 	# Write events to WAL file
+	# 	content = '\n'.join(json.dumps(event) for event in events) + '\n'
+	# 	await anyio.Path(wal_path).write_text(content)
+
+	# 	# Call the method under test (temp_config_dir fixture already sets the env var)
+	# 	await service._update_wal_user_ids(service.session_id)
+
+	# 	# Read back the updated file and verify changes
+	# 	content = await anyio.Path(wal_path).read_text()
+
+	# 	updated_events = []
+	# 	for line in content.splitlines():
+	# 		if line.strip():
+	# 			updated_events.append(json.loads(line))
+
+	# 	# Verify all user_ids were updated to the authenticated user's ID
+	# 	assert len(updated_events) == 3
+	# 	for event in updated_events:
+	# 		assert event['user_id'] == 'test-user-123'
+
+	# 	# Verify other fields remained unchanged
+	# 	assert updated_events[0]['event_type'] == 'CreateAgentTaskEvent'
+	# 	assert updated_events[0]['task'] == 'Task 1'
+	# 	assert updated_events[1]['event_type'] == 'UpdateAgentTaskEvent'
+	# 	assert updated_events[1]['status'] == 'done'
+	# 	assert updated_events[2]['event_type'] == 'CreateAgentStepEvent'
+	# 	assert updated_events[2]['step'] == 1
+
+
+class TestIntegration:
+	"""Integration tests for OAuth2 and cloud sync."""
+
+	async def test_full_auth_flow(self, httpserver: HTTPServer, temp_config_dir):
+		"""Test complete authentication flow."""
+		# Track token polling attempts
+		token_attempts = 0
+
+		def handle_token_request(request):
+			nonlocal token_attempts
+			token_attempts += 1
+
+			from werkzeug.wrappers import Response
+
+			if token_attempts == 1:
+				# First attempt: pending
+				return Response(
+					json.dumps({'error': 'authorization_pending'}),
+					status=200,
+					mimetype='application/json',
+				)
+			else:
+				# Second attempt: success
+				return Response(
+					json.dumps(
+						{
+							'access_token': 'test-api-key',
+							'token_type': 'Bearer',
+							'user_id': 'test-user-123',
+						}
+					),
+					status=200,
+					mimetype='application/json',
+				)
+
+		# Set up auth flow endpoints
+		httpserver.expect_request(
+			'/api/v1/oauth/device/authorize',
+			method='POST',
+		).respond_with_json(
+			{
+				'device_code': 'test-device-code',
+				'user_code': 'ABCD-1234',
+				'verification_uri': f'{httpserver.url_for("")}/device',
+				'verification_uri_complete': f'{httpserver.url_for("")}/device?user_code=ABCD-1234',
+				'expires_in': 1800,
+				'interval': 0.1,  # Fast polling for test
+			}
+		)
+
+		httpserver.expect_request(
+			'/api/v1/oauth/device/token',
+			method='POST',
+		).respond_with_handler(handle_token_request)
+
+		# Set up events endpoint
+		httpserver.expect_request(
+			'/api/v1/events',
+			method='POST',
+		).respond_with_json({'processed': 1, 'failed': 0})
+
+		# Create service
+		service = CloudSync(base_url=httpserver.url_for(''))
+		service.session_id = 'test-session-id'
+
+		# Send pre-auth event
+		await service.handle_event(
+			CreateAgentSessionEvent(
+				user_id=TEMP_USER_ID,
+				browser_session_id='test-browser-session',
+				browser_session_live_url='http://example.com/live',
+				browser_session_cdp_url='ws://example.com/cdp',
+				device_id='test-device-id',
+			)
+		)
+
+		# Authenticate
+		authenticated = await service.authenticate(show_instructions=False)
+		assert authenticated is True
+		assert service.auth_client is not None
+		assert service.auth_client.is_authenticated
+		assert service.auth_client.api_token == 'test-api-key'
+		assert service.auth_client.user_id == 'test-user-123'
+
+		# Send authenticated event
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Authenticated task',
+				user_id='test-user-123',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+				device_id='test-device-id',
+			)
+		)
+
+		# Verify auth was saved
+		auth_file = temp_config_dir / 'cloud_auth.json'
+		assert await anyio.Path(auth_file).exists()
+
+		content = await anyio.Path(auth_file).read_text()
+		saved_auth = json.loads(content)
+		assert saved_auth['api_token'] == 'test-api-key'
+		assert saved_auth['user_id'] == 'test-user-123'
+
+
+class TestAuthResilience:
+	"""Test auth resilience scenarios - agent should never break due to sync failures."""
+
+	async def test_token_expiry_handling(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Test that expired tokens are handled gracefully."""
+		# Set up successful auth flow first
+		httpserver.expect_request(
+			'/api/v1/oauth/device/authorize',
+			method='POST',
+		).respond_with_json(
+			{
+				'device_code': 'test-device-code',
+				'user_code': 'ABCD-1234',
+				'verification_uri': 'https://example.com/device',
+				'verification_uri_complete': 'https://example.com/device?user_code=ABCD-1234',
+				'expires_in': 1800,
+				'interval': 0.1,
+			}
+		)
+
+		httpserver.expect_request(
+			'/api/v1/oauth/device/token',
+			method='POST',
+		).respond_with_json(
+			{
+				'access_token': 'test-api-key',
+				'token_type': 'Bearer',
+				'user_id': 'test-user-123',
+				'scope': 'read write',
+			}
+		)
+
+		# Authenticate successfully first
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+		success = await auth.authenticate(agent_session_id='test-session-id', show_instructions=False)
+		assert success is True
+
+		# Now simulate token expiry by returning 401 errors
+		httpserver.expect_request(
+			'/api/v1/events',
+			method='POST',
+		).respond_with_json({'error': 'unauthorized', 'detail': 'Token expired'}, status=401)
+
+		# Create cloud sync service
+		from browser_use.sync.service import CloudSync
+
+		service = CloudSync(base_url=httpserver.url_for(''))
+		service.auth_client = auth
+
+		# Send event - should not raise exception even though token is expired
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Test task after token expiry',
+				user_id='test-user-123',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+				device_id='test-device-id',
+			)
+		)
+
+		# Agent should continue functioning despite sync failure
+		assert True  # No exception raised
+
+	async def test_auth_failure_resilience(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Test that auth failures don't break the agent."""
+		# Set up auth endpoint to always fail
+		httpserver.expect_request(
+			'/api/v1/oauth/device/authorize',
+			method='POST',
+		).respond_with_json({'error': 'invalid_client', 'error_description': 'Client not found'}, status=400)
+
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+
+		# Auth should fail gracefully without throwing
+		success = await auth.authenticate(agent_session_id='test-session-id', show_instructions=False)
+		assert success is False
+
+		# Should still be able to create sync service
+		from browser_use.sync.service import CloudSync
+
+		service = CloudSync(base_url=httpserver.url_for(''))
+		service.auth_client = auth
+
+		# Set up events endpoint to handle unauthenticated requests
+		httpserver.expect_request(
+			'/api/v1/events',
+			method='POST',
+		).respond_with_json({'processed': 1, 'failed': 0})
+
+		# Should be able to send events without auth (pre-auth mode)
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Test task without auth',
+				user_id='',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+				device_id='test-device-id',
+			)
+		)
+
+	async def test_server_downtime_resilience(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Test that server downtime doesn't break the agent."""
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+
+		# Don't set up any server responses - simulate server being down
+
+		# Auth should timeout gracefully
+		result = await auth.poll_for_token('fake-device-code', interval=0.1, timeout=0.3)
+		assert result is None
+
+		from browser_use.sync.service import CloudSync
+
+		service = CloudSync(base_url=httpserver.url_for(''))
+		service.auth_client = auth
+
+		# Should be able to send events even when server is down
+		# They will be queued locally
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Test task during server downtime',
+				user_id='test-user-123',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+				device_id='test-device-id',
+			)
+		)
+
+	async def test_excessive_event_queue_handling(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Test that excessive event queuing doesn't break the agent."""
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+
+		from browser_use.sync.service import CloudSync
+
+		service = CloudSync(base_url=httpserver.url_for(''))
+		service.auth_client = auth
+
+		# Send many events while server is down (no responses configured)
+		for i in range(100):
+			await service.handle_event(
+				CreateAgentTaskEvent(
+					agent_session_id='test-session',
+					llm_model='test-model',
+					task=f'Test task {i}',
+					user_id='test-user-123',
+					done_output=None,
+					user_feedback_type=None,
+					user_comment=None,
+					gif_url=None,
+					device_id='test-device-id',
+				)
+			)
+
+		# Agent should still be functioning
+		assert True  # No memory issues or crashes
+
+	async def test_malformed_server_responses(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Test that malformed server responses don't break the agent."""
+		# Set up malformed JSON responses
+		httpserver.expect_request(
+			'/api/v1/oauth/device/authorize',
+			method='POST',
+		).respond_with_data('invalid json{', status=200, content_type='application/json')
+
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+
+		# Should handle malformed response gracefully
+		try:
+			await auth.start_device_authorization('test-session-id')
+		except Exception:
+			pass  # Exception is expected but shouldn't crash the agent
+
+		# Set up another malformed response for events
+		httpserver.expect_request(
+			'/api/v1/events',
+			method='POST',
+		).respond_with_data('malformed response', status=500)
+
+		from browser_use.sync.service import CloudSync
+
+		service = CloudSync(base_url=httpserver.url_for(''))
+		service.auth_client = auth
+
+		# Should handle malformed event response gracefully
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Test task with malformed response',
+				user_id='test-user-123',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+				device_id='test-device-id',
+			)
+		)

chapter9/browser-use-rpa/browser-use/tests/ci/test_sync_client_legacy.py.disabled

@@ -0,0 +1,603 @@
+"""Tests for CloudSync client machinery - retry logic, event handling, backend communication."""
+
+import os
+import tempfile
+from pathlib import Path
+
+import httpx
+import pytest
+from bubus import BaseEvent
+from pytest_httpserver import HTTPServer
+
+from browser_use.agent.cloud_events import CreateAgentTaskEvent
+from browser_use.sync.auth import TEMP_USER_ID, DeviceAuthClient
+from browser_use.sync.service import CloudSync
+
+
+@pytest.fixture
+def temp_config_dir():
+	"""Create temporary config directory for tests."""
+	with tempfile.TemporaryDirectory() as tmpdir:
+		temp_dir = Path(tmpdir) / '.config' / 'browseruse'
+		temp_dir.mkdir(parents=True, exist_ok=True)
+
+		os.environ['BROWSER_USE_CONFIG_DIR'] = str(temp_dir)
+
+		yield temp_dir
+
+
+@pytest.fixture
+async def http_client(httpserver: HTTPServer):
+	"""Create a real HTTP client pointed at the test server."""
+	async with httpx.AsyncClient(base_url=httpserver.url_for('')) as client:
+		yield client
+
+
+class TestCloudSyncInit:
+	"""Test CloudSync initialization and configuration."""
+
+	async def test_init_with_auth_enabled(self, temp_config_dir):
+		"""Test CloudSync initialization with auth enabled."""
+		service = CloudSync(base_url='http://localhost:8000')
+
+		assert service.base_url == 'http://localhost:8000'
+		assert service.auth_client is not None
+		assert isinstance(service.auth_client, DeviceAuthClient)
+		assert service.session_id is None
+
+	async def test_init_basic(self, temp_config_dir):
+		"""Test CloudSync basic initialization."""
+		service = CloudSync(base_url='http://localhost:8000')
+
+		assert service.base_url == 'http://localhost:8000'
+		assert service.auth_client is not None  # Always has auth_client in current version
+		assert isinstance(service.auth_client, DeviceAuthClient)
+
+
+class TestCloudSyncEventHandling:
+	"""Test CloudSync event validation and processing."""
+
+	@pytest.fixture
+	def authenticated_sync(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Create authenticated CloudSync service."""
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+		auth.auth_config.api_token = 'test-api-key'
+		auth.auth_config.user_id = 'test-user-123'
+
+		service = CloudSync(base_url=httpserver.url_for(''), enable_auth=True)
+		service.auth_client = auth
+		service.session_id = 'test-session-id'
+		return service
+
+	@pytest.fixture
+	def unauthenticated_sync(self, httpserver: HTTPServer, temp_config_dir):
+		"""Create unauthenticated CloudSync service."""
+		service = CloudSync(base_url=httpserver.url_for(''), enable_auth=True)
+		service.session_id = 'test-session-id'
+		return service
+
+	async def test_event_forwarding_authenticated(self, httpserver: HTTPServer, authenticated_sync):
+		"""Test event forwarding when authenticated."""
+		# Capture requests
+		requests = []
+
+		def capture_request(request):
+			requests.append(request.get_json())
+			from werkzeug.wrappers import Response
+
+			return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
+
+		# Send event
+		await authenticated_sync.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Test task',
+				user_id='test-user-123',
+				device_id='test-device-id',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+			)
+		)
+
+		# Verify forwarding
+		assert len(requests) == 1
+		event_batch = requests[0]
+		assert len(event_batch['events']) == 1
+
+		event = event_batch['events'][0]
+		assert event['event_type'] == 'CreateAgentTaskEvent'
+		assert event['user_id'] == 'test-user-123'
+		# BaseEvent creates event_type attribute, plus our custom data as attributes
+		assert event['task'] == 'Test task'
+
+	async def test_event_queueing_unauthenticated(self, httpserver: HTTPServer, unauthenticated_sync):
+		"""Test event queueing when unauthenticated."""
+		# Server returns 401
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_json({'error': 'unauthorized'}, status=401)
+
+		# Send event
+		await unauthenticated_sync.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Queued task',
+				user_id=TEMP_USER_ID,
+				device_id='test-device-id',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+			)
+		)
+
+		# Event should be queued
+		assert len(unauthenticated_sync.pending_events) == 1
+		queued_event = unauthenticated_sync.pending_events[0]
+		assert queued_event.event_type == 'CreateAgentTaskEvent'
+		assert queued_event.user_id == TEMP_USER_ID
+		assert queued_event.task == 'Queued task'
+
+	async def test_event_user_id_injection_pre_auth(self, httpserver: HTTPServer, unauthenticated_sync):
+		"""Test that temp user ID is injected for pre-auth events."""
+		requests = []
+
+		def capture_request(request):
+			requests.append(request.get_json())
+			from werkzeug.wrappers import Response
+
+			return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
+
+		# Send event without user_id
+		await unauthenticated_sync.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Pre-auth task',
+				user_id=TEMP_USER_ID,
+				device_id='test-device-id',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+			)
+		)
+
+		# Verify temp user ID was injected
+		assert len(requests) == 1
+		event = requests[0]['events'][0]
+		assert event['user_id'] == TEMP_USER_ID
+
+
+class TestCloudSyncRetryLogic:
+	"""Test CloudSync retry and error handling logic."""
+
+	@pytest.fixture
+	def sync_with_auth(self, httpserver: HTTPServer, http_client, temp_config_dir):
+		"""Create CloudSync with auth."""
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
+		auth.auth_config.api_token = 'test-api-key'
+		auth.auth_config.user_id = 'test-user-123'
+
+		service = CloudSync(base_url=httpserver.url_for(''), enable_auth=True)
+		service.auth_client = auth
+		service.session_id = 'test-session-id'
+		return service
+
+	async def test_pending_event_resending(self, httpserver: HTTPServer, sync_with_auth):
+		"""Test resending of pending events after authentication."""
+		requests = []
+
+		def capture_request(request):
+			requests.append(request.get_json())
+			from werkzeug.wrappers import Response
+
+			return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
+
+		# Manually add pending events (simulating 401 scenario)
+		sync_with_auth.pending_events.extend(
+			[
+				CreateAgentTaskEvent(
+					agent_session_id='test-session',
+					llm_model='test-model',
+					task='Pending task 1',
+					user_id=TEMP_USER_ID,
+					device_id='test-device-id',
+					done_output=None,
+					user_feedback_type=None,
+					user_comment=None,
+					gif_url=None,
+				),
+				CreateAgentTaskEvent(
+					agent_session_id='test-session',
+					llm_model='test-model',
+					task='Pending task 2',
+					user_id=TEMP_USER_ID,
+					device_id='test-device-id',
+					done_output=None,
+					user_feedback_type=None,
+					user_comment=None,
+					gif_url=None,
+				),
+			]
+		)
+
+		# Resend pending events
+		await sync_with_auth._resend_pending_events()
+
+		# Should send all pending events with updated user ID
+		assert len(requests) == 2
+		for i, request in enumerate(requests):
+			event = request['events'][0]
+			assert event['user_id'] == 'test-user-123'  # Updated from temp ID
+			assert f'Pending task {i + 1}' == event['task']
+
+		# Pending events should be cleared
+		assert len(sync_with_auth.pending_events) == 0
+
+	async def test_backend_error_resilience(self, httpserver: HTTPServer, sync_with_auth):
+		"""Test resilience to backend errors."""
+		# Server returns 500 error
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_data('Internal Server Error', status=500)
+
+		# Should not raise exception
+		await sync_with_auth.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Task during outage',
+				user_id='test-user-123',
+				device_id='test-device-id',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+			)
+		)
+
+		# Events should not be queued for 500 errors (only 401)
+		assert len(sync_with_auth.pending_events) == 0
+
+	async def test_network_error_resilience(self, sync_with_auth):
+		"""Test resilience to network errors."""
+		# No server running - will get connection error
+		sync_with_auth.base_url = 'http://localhost:99999'  # Invalid port
+
+		# Should not raise exception
+		await sync_with_auth.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Task during network error',
+				user_id='test-user-123',
+				device_id='test-device-id',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+			)
+		)
+
+		# Should handle gracefully without crashing
+
+	async def test_concurrent_event_sending(self, httpserver: HTTPServer, sync_with_auth):
+		"""Test handling of concurrent event sending."""
+		import asyncio
+
+		requests = []
+
+		def capture_request(request):
+			requests.append(request.get_json())
+			from werkzeug.wrappers import Response
+
+			return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
+
+		# Send multiple events concurrently
+		tasks = []
+		for i in range(5):
+			task = sync_with_auth.handle_event(
+				CreateAgentTaskEvent(
+					agent_session_id='test-session',
+					llm_model='test-model',
+					task=f'Concurrent task {i}',
+					user_id='test-user-123',
+					device_id='test-device-id',
+					done_output=None,
+					user_feedback_type=None,
+					user_comment=None,
+					gif_url=None,
+				)
+			)
+			tasks.append(task)
+
+		await asyncio.gather(*tasks)
+
+		# All events should be sent
+		assert len(requests) == 5
+		# Just verify all events have task data - order may vary due to concurrency
+		task_values = [req['events'][0]['task'] for req in requests]
+		expected_tasks = [f'Concurrent task {i}' for i in range(5)]
+		assert sorted(task_values) == sorted(expected_tasks)
+
+
+class TestCloudSyncBackendCommunication:
+	"""Test CloudSync backend communication patterns."""
+
+	async def test_request_format_validation(self, httpserver: HTTPServer, temp_config_dir):
+		"""Test that requests are formatted correctly for backend."""
+		requests = []
+
+		def capture_request(request):
+			# Validate request structure
+			assert request.content_type == 'application/json'
+			data = request.get_json()
+			requests.append(data)
+
+			# Validate batch structure
+			assert 'events' in data
+			assert isinstance(data['events'], list)
+			assert len(data['events']) == 1
+
+			event = data['events'][0]
+			required_fields = ['event_type', 'event_id', 'event_created_at', 'event_schema', 'user_id']
+			for field in required_fields:
+				assert field in event, f'Missing required field: {field}'
+
+			from werkzeug.wrappers import Response
+
+			return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
+
+		# Create authenticated service
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+		auth.auth_config.api_token = 'test-api-key'
+		auth.auth_config.user_id = 'test-user-123'
+
+		service = CloudSync(base_url=httpserver.url_for(''), enable_auth=True)
+		service.auth_client = auth
+		service.session_id = 'test-session-id'
+
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Format validation test',
+				user_id='test-user-123',
+				device_id='test-device-id',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+			)
+		)
+
+		assert len(requests) == 1
+
+	async def test_auth_header_handling(self, httpserver: HTTPServer, temp_config_dir):
+		"""Test proper auth header handling."""
+		requests = []
+
+		def capture_request(request):
+			requests.append(
+				{
+					'headers': dict(request.headers),
+					'json': request.get_json(),
+				}
+			)
+			from werkzeug.wrappers import Response
+
+			return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
+
+		# Test authenticated request
+		auth = DeviceAuthClient(base_url=httpserver.url_for(''))
+		auth.auth_config.api_token = 'test-api-key'
+		auth.auth_config.user_id = 'test-user-123'
+
+		service = CloudSync(base_url=httpserver.url_for(''), enable_auth=True)
+		service.auth_client = auth
+
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Auth header test',
+				user_id='test-user-123',
+				device_id='test-device-id',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+			)
+		)
+
+		# Check auth header was included
+		assert len(requests) == 1
+		headers = requests[0]['headers']
+		assert 'Authorization' in headers
+		assert headers['Authorization'] == 'Bearer test-api-key'
+
+		# Test unauthenticated request
+		requests.clear()
+		service.auth_client = DeviceAuthClient(base_url=httpserver.url_for(''))  # No credentials
+
+		await service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='No auth test',
+				user_id='',
+				device_id='test-device-id',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+			)
+		)
+
+		# Check no auth header
+		assert len(requests) == 1
+		headers = requests[0]['headers']
+		assert 'Authorization' not in headers
+
+
+class TestCloudSyncErrorHandling:
+	"""Test CloudSync error handling doesn't crash the agent."""
+
+	@pytest.fixture
+	def sync_service(self, httpserver: HTTPServer, temp_config_dir):
+		"""Create CloudSync service."""
+		return CloudSync(base_url=httpserver.url_for(''), enable_auth=False)
+
+	async def test_timeout_error_handling(self, sync_service):
+		"""Test that timeout errors are handled gracefully."""
+		# Use a URL that will timeout
+		sync_service.base_url = 'http://10.255.255.1'  # Non-routable IP for timeout
+
+		# Should not raise exception
+		await sync_service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Timeout test',
+				user_id='test-user-123',
+				device_id='test-device-id',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+			)
+		)
+
+	async def test_malformed_event_handling(self, httpserver: HTTPServer, sync_service):
+		"""Test handling of events that can't be serialized."""
+
+		class BadEvent(BaseEvent):
+			"""Event that will fail to serialize."""
+
+			event_type: str = 'BadEvent'
+
+			def model_dump(self, **kwargs):
+				raise ValueError('Serialization failed')
+
+		# Should not raise exception
+		await sync_service.handle_event(BadEvent())
+
+	async def test_http_error_responses(self, httpserver: HTTPServer, sync_service):
+		"""Test various HTTP error responses don't crash the service."""
+		error_codes = [400, 403, 404, 429, 500, 502, 503]
+
+		for status_code in error_codes:
+			httpserver.expect_request('/api/v1/events', method='POST').respond_with_json(
+				{'error': f'Test error {status_code}'}, status=status_code
+			)
+
+			# Should not raise exception
+			await sync_service.handle_event(
+				CreateAgentTaskEvent(
+					agent_session_id='test-session',
+					llm_model='test-model',
+					task=f'Error {status_code} test',
+					user_id='test-user-123',
+					device_id='test-device-id',
+					done_output=None,
+					user_feedback_type=None,
+					user_comment=None,
+					gif_url=None,
+				)
+			)
+
+	async def test_invalid_response_handling(self, httpserver: HTTPServer, sync_service):
+		"""Test handling of invalid server responses."""
+		# Return invalid JSON
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_data('Not JSON', status=200)
+
+		# Should not raise exception
+		await sync_service.handle_event(
+			CreateAgentTaskEvent(
+				agent_session_id='test-session',
+				llm_model='test-model',
+				task='Invalid response test',
+				user_id='test-user-123',
+				device_id='test-device-id',
+				done_output=None,
+				user_feedback_type=None,
+				user_comment=None,
+				gif_url=None,
+			)
+		)
+
+	async def test_event_with_restricted_attributes(self, httpserver: HTTPServer, sync_service):
+		"""Test handling events that don't allow user_id attribute."""
+		from pydantic import ConfigDict
+
+		class RestrictedEvent(BaseEvent):
+			"""Event that doesn't allow extra attributes."""
+
+			model_config = ConfigDict(extra='forbid')
+			event_type: str = 'RestrictedEvent'
+			data: str = 'test'
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_json({'processed': 1}, status=200)
+
+		# Should not raise exception - will log debug message about not being able to set user_id
+		await sync_service.handle_event(RestrictedEvent())
+
+	async def test_concurrent_error_resilience(self, httpserver: HTTPServer, sync_service):
+		"""Test that concurrent errors don't affect other events."""
+		import asyncio
+
+		successful_requests = []
+		request_count = 0
+
+		def handler(request):
+			nonlocal request_count
+			request_count += 1
+			# Every 3rd request fails
+			if request_count % 3 == 0:
+				from werkzeug.wrappers import Response
+
+				return Response('Server Error', status=500)
+			else:
+				successful_requests.append(request.get_json())
+				from werkzeug.wrappers import Response
+
+				return Response('{"processed": 1}', status=200, mimetype='application/json')
+
+		httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(handler)
+
+		# Send 10 events concurrently
+		tasks = []
+		for i in range(10):
+			task = sync_service.handle_event(
+				CreateAgentTaskEvent(
+					agent_session_id='test-session',
+					llm_model='test-model',
+					task=f'Concurrent error test {i}',
+					user_id='test-user-123',
+					device_id='test-device-id',
+					done_output=None,
+					user_feedback_type=None,
+					user_comment=None,
+					gif_url=None,
+				)
+			)
+			tasks.append(task)
+
+		# All should complete without raising
+		await asyncio.gather(*tasks)
+
+		# ~7 should succeed (10 total, ~3 fail)
+		assert len(successful_requests) >= 6

chapter9/browser-use-rpa/browser-use/tests/ci/test_telemetry.py

@@ -0,0 +1,286 @@
+"""Test telemetry functionality."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from browser_use.config import CONFIG
+from browser_use.telemetry import (
+	CLITelemetryEvent,
+	MCPClientTelemetryEvent,
+	MCPServerTelemetryEvent,
+	ProductTelemetry,
+)
+from browser_use.utils import get_browser_use_version
+
+
+@pytest.fixture
+def reset_telemetry_singleton():
+	"""Reset the telemetry singleton between tests."""
+	# The singleton decorator stores instance in a list at index 0
+	# We need to access the closure variable which stores the singleton instance
+	# Import the actual module to access the wrapped function
+	from browser_use.telemetry import service
+
+	# Get the ProductTelemetry wrapper function created by @singleton
+	wrapper_func = service.ProductTelemetry
+
+	# Access the closure variable (instance list) and reset it
+	# The closure contains the 'instance' list at index 0
+	# Type ignore needed because pyright doesn't know this is a wrapper function
+	if hasattr(wrapper_func, '__closure__') and wrapper_func.__closure__:  # type: ignore
+		for cell in wrapper_func.__closure__:  # type: ignore
+			if hasattr(cell.cell_contents, '__setitem__'):
+				# This is the instance list
+				cell.cell_contents[0] = None
+				break
+
+	yield
+
+	# Reset again after test
+	if hasattr(wrapper_func, '__closure__') and wrapper_func.__closure__:  # type: ignore
+		for cell in wrapper_func.__closure__:  # type: ignore
+			if hasattr(cell.cell_contents, '__setitem__'):
+				cell.cell_contents[0] = None
+				break
+
+
+@pytest.fixture
+def mock_posthog():
+	"""Mock PostHog client."""
+	with patch('browser_use.telemetry.service.Posthog') as mock:
+		yield mock
+
+
+def test_telemetry_disabled_when_config_false(monkeypatch, reset_telemetry_singleton):
+	"""Test that telemetry is disabled when ANONYMIZED_TELEMETRY is False."""
+	# Set env var to disable telemetry
+	monkeypatch.setattr(CONFIG, 'ANONYMIZED_TELEMETRY', False)
+
+	# Create telemetry instance
+	telemetry = ProductTelemetry()
+
+	# Check that posthog client is None
+	assert telemetry._posthog_client is None
+
+	# Try to capture an event - should not fail
+	event = CLITelemetryEvent(
+		version=get_browser_use_version(),
+		action='start',
+		mode='interactive',
+	)
+	telemetry.capture(event)  # Should not raise
+
+
+def test_telemetry_enabled_when_config_true(monkeypatch, mock_posthog, reset_telemetry_singleton):
+	"""Test that telemetry is enabled when ANONYMIZED_TELEMETRY is True."""
+	# Set env var to enable telemetry
+	monkeypatch.setattr(CONFIG, 'ANONYMIZED_TELEMETRY', True)
+
+	# Create telemetry instance
+	telemetry = ProductTelemetry()
+
+	# Check that posthog client is created
+	assert telemetry._posthog_client is not None
+	mock_posthog.assert_called_once()
+
+
+def test_cli_telemetry_event():
+	"""Test CLITelemetryEvent structure."""
+	event = CLITelemetryEvent(
+		version='1.0.0',
+		action='start',
+		mode='interactive',
+		model='gpt-4o',
+		model_provider='OpenAI',
+		duration_seconds=10.5,
+		error_message=None,
+	)
+
+	assert event.name == 'cli_event'
+	assert event.version == '1.0.0'
+	assert event.action == 'start'
+	assert event.mode == 'interactive'
+	assert event.model == 'gpt-4o'
+	assert event.model_provider == 'OpenAI'
+	assert event.duration_seconds == 10.5
+	assert event.error_message is None
+
+	# Check properties
+	props = event.properties
+	assert 'version' in props
+	assert 'action' in props
+	assert 'mode' in props
+	assert 'is_docker' in props  # Docker context should be included
+	assert isinstance(props['is_docker'], bool)  # Should be a boolean
+	assert 'name' not in props  # name should not be in properties
+
+
+def test_mcp_client_telemetry_event():
+	"""Test MCPClientTelemetryEvent structure."""
+	event = MCPClientTelemetryEvent(
+		server_name='test-server',
+		command='npx',
+		tools_discovered=5,
+		version='1.0.0',
+		action='connect',
+		tool_name='browser_navigate',
+		duration_seconds=2.5,
+		error_message=None,
+	)
+
+	assert event.name == 'mcp_client_event'
+	assert event.server_name == 'test-server'
+	assert event.command == 'npx'
+	assert event.tools_discovered == 5
+	assert event.version == '1.0.0'
+	assert event.action == 'connect'
+	assert event.tool_name == 'browser_navigate'
+	assert event.duration_seconds == 2.5
+	assert event.error_message is None
+
+
+def test_mcp_server_telemetry_event():
+	"""Test MCPServerTelemetryEvent structure."""
+	event = MCPServerTelemetryEvent(
+		version='1.0.0',
+		action='start',
+		tool_name='browser_click',
+		duration_seconds=1.2,
+		error_message='Test error',
+	)
+
+	assert event.name == 'mcp_server_event'
+	assert event.version == '1.0.0'
+	assert event.action == 'start'
+	assert event.tool_name == 'browser_click'
+	assert event.duration_seconds == 1.2
+	assert event.error_message == 'Test error'
+
+
+def test_telemetry_capture_with_mock(monkeypatch, reset_telemetry_singleton):
+	"""Test telemetry capture with mocked PostHog client."""
+	# Enable telemetry
+	monkeypatch.setattr(CONFIG, 'ANONYMIZED_TELEMETRY', True)
+
+	# Create mock posthog client
+	mock_client = MagicMock()
+
+	# Create telemetry instance and inject mock
+	telemetry = ProductTelemetry()
+	telemetry._posthog_client = mock_client
+
+	# Capture an event
+	event = CLITelemetryEvent(
+		version='1.0.0',
+		action='start',
+		mode='oneshot',
+	)
+	telemetry.capture(event)
+
+	# Check that capture was called
+	mock_client.capture.assert_called_once()
+	call_args = mock_client.capture.call_args
+	assert call_args[1]['event'] == 'cli_event'
+	assert 'properties' in call_args[1]
+	assert call_args[1]['properties']['version'] == '1.0.0'
+	assert call_args[1]['properties']['action'] == 'start'
+	assert call_args[1]['properties']['mode'] == 'oneshot'
+
+
+def test_telemetry_flush(monkeypatch, reset_telemetry_singleton):
+	"""Test telemetry flush method."""
+	# Enable telemetry
+	monkeypatch.setattr(CONFIG, 'ANONYMIZED_TELEMETRY', True)
+
+	# Create mock posthog client
+	mock_client = MagicMock()
+
+	# Create telemetry instance and inject mock
+	telemetry = ProductTelemetry()
+	telemetry._posthog_client = mock_client
+
+	# Call flush
+	telemetry.flush()
+
+	# Check that flush was called
+	mock_client.flush.assert_called_once()
+
+
+def test_telemetry_user_id_generation(tmp_path, monkeypatch, reset_telemetry_singleton):
+	"""Test that telemetry generates and persists user ID."""
+	# Set BROWSER_USE_CONFIG_DIR to temp directory
+	config_dir = tmp_path / 'config' / 'browseruse'
+	config_dir.mkdir(parents=True)
+	monkeypatch.setenv('BROWSER_USE_CONFIG_DIR', str(config_dir))
+
+	# Enable telemetry
+	monkeypatch.setattr(CONFIG, 'ANONYMIZED_TELEMETRY', True)
+
+	# Create telemetry instance with patched path
+	telemetry1 = ProductTelemetry()
+	# Manually patch the USER_ID_PATH on the instance
+	telemetry1.USER_ID_PATH = str(config_dir / 'device_id')
+
+	user_id1 = telemetry1.user_id
+
+	# Check that user ID is generated
+	assert user_id1 != 'UNKNOWN_USER_ID'
+	assert len(user_id1) > 0
+
+	# Create another instance - should get same ID
+	telemetry2 = ProductTelemetry()
+	telemetry2.USER_ID_PATH = str(config_dir / 'device_id')
+	user_id2 = telemetry2.user_id
+
+	assert user_id1 == user_id2
+
+	# Check that ID was persisted in config directory
+	id_file = config_dir / 'device_id'
+	assert id_file.exists()
+	assert id_file.read_text() == user_id1
+
+
+def test_mcp_server_telemetry_event_with_parent_process():
+	"""Test MCPServerTelemetryEvent with parent_process_cmdline field."""
+	event = MCPServerTelemetryEvent(
+		version='1.0.0',
+		action='start',
+		tool_name=None,
+		duration_seconds=None,
+		error_message=None,
+		parent_process_cmdline='python -m browser_use.mcp.server',
+	)
+
+	assert event.name == 'mcp_server_event'
+	assert event.version == '1.0.0'
+	assert event.action == 'start'
+	assert event.parent_process_cmdline == 'python -m browser_use.mcp.server'
+
+	# Check properties includes parent_process_cmdline
+	props = event.properties
+	assert 'parent_process_cmdline' in props
+	assert props['parent_process_cmdline'] == 'python -m browser_use.mcp.server'
+	assert 'is_docker' in props  # Docker context should be included
+	assert isinstance(props['is_docker'], bool)  # Should be a boolean
+
+
+def test_telemetry_device_id_uses_config_dir():
+	"""Test that telemetry device_id is stored in config directory, not cache directory."""
+	# This test verifies that the ProductTelemetry class uses CONFIG.BROWSER_USE_CONFIG_DIR
+	# for the device_id file location instead of the cache directory.
+
+	# Import ProductTelemetry to check the path
+	from browser_use.telemetry.service import ProductTelemetry
+
+	# Create telemetry instance
+	telemetry = ProductTelemetry()
+
+	# The USER_ID_PATH should use the config directory
+	# We check that it contains the config dir path and ends with 'device_id'
+	assert 'browseruse/device_id' in telemetry.USER_ID_PATH or 'browseruse\\device_id' in telemetry.USER_ID_PATH
+	assert telemetry.USER_ID_PATH.endswith('device_id')
+
+	# Verify it's not using the cache directory path
+	assert 'cache' not in telemetry.USER_ID_PATH
+	assert 'telemetry_user_id' not in telemetry.USER_ID_PATH

chapter9/browser-use-rpa/browser-use/tests/ci/test_tools.py

@@ -0,0 +1,582 @@
+import asyncio
+import tempfile
+import time
+
+import pytest
+from pydantic import BaseModel
+from pytest_httpserver import HTTPServer
+
+from browser_use.agent.views import ActionModel, ActionResult
+from browser_use.browser import BrowserSession
+from browser_use.browser.profile import BrowserProfile
+from browser_use.filesystem.file_system import FileSystem
+from browser_use.tools.service import Tools
+from browser_use.tools.views import (
+	DoneAction,
+	GoToUrlAction,
+	NoParamsAction,
+	SearchGoogleAction,
+)
+
+
+@pytest.fixture(scope='session')
+def http_server():
+	"""Create and provide a test HTTP server that serves static content."""
+	server = HTTPServer()
+	server.start()
+
+	# Add routes for common test pages
+	server.expect_request('/').respond_with_data(
+		'<html><head><title>Test Home Page</title></head><body><h1>Test Home Page</h1><p>Welcome to the test site</p></body></html>',
+		content_type='text/html',
+	)
+
+	server.expect_request('/page1').respond_with_data(
+		'<html><head><title>Test Page 1</title></head><body><h1>Test Page 1</h1><p>This is test page 1</p></body></html>',
+		content_type='text/html',
+	)
+
+	server.expect_request('/page2').respond_with_data(
+		'<html><head><title>Test Page 2</title></head><body><h1>Test Page 2</h1><p>This is test page 2</p></body></html>',
+		content_type='text/html',
+	)
+
+	server.expect_request('/search').respond_with_data(
+		"""
+		<html>
+		<head><title>Search Results</title></head>
+		<body>
+			<h1>Search Results</h1>
+			<div class="results">
+				<div class="result">Result 1</div>
+				<div class="result">Result 2</div>
+				<div class="result">Result 3</div>
+			</div>
+		</body>
+		</html>
+		""",
+		content_type='text/html',
+	)
+
+	yield server
+	server.stop()
+
+
+@pytest.fixture(scope='session')
+def base_url(http_server):
+	"""Return the base URL for the test HTTP server."""
+	return f'http://{http_server.host}:{http_server.port}'
+
+
+@pytest.fixture(scope='module')
+async def browser_session():
+	"""Create and provide a Browser instance with security disabled."""
+	browser_session = BrowserSession(
+		browser_profile=BrowserProfile(
+			headless=True,
+			user_data_dir=None,
+			keep_alive=True,
+		)
+	)
+	await browser_session.start()
+	yield browser_session
+	await browser_session.kill()
+
+
+@pytest.fixture(scope='function')
+def tools():
+	"""Create and provide a Tools instance."""
+	return Tools()
+
+
+class TestToolsIntegration:
+	"""Integration tests for Tools using actual browser instances."""
+
+	async def test_registry_actions(self, tools, browser_session):
+		"""Test that the registry contains the expected default actions."""
+		# Check that common actions are registered
+		common_actions = [
+			'go_to_url',
+			'search_google',
+			'click_element_by_index',
+			'input_text',
+			'scroll',
+			'go_back',
+			'switch_tab',
+			'close_tab',
+			'wait',
+		]
+
+		for action in common_actions:
+			assert action in tools.registry.registry.actions
+			assert tools.registry.registry.actions[action].function is not None
+			assert tools.registry.registry.actions[action].description is not None
+
+	async def test_custom_action_registration(self, tools, browser_session, base_url):
+		"""Test registering a custom action and executing it."""
+
+		# Define a custom action
+		class CustomParams(BaseModel):
+			text: str
+
+		@tools.action('Test custom action', param_model=CustomParams)
+		async def custom_action(params: CustomParams, browser_session):
+			current_url = await browser_session.get_current_page_url()
+			return ActionResult(extracted_content=f'Custom action executed with: {params.text} on {current_url}')
+
+		# Navigate to a page first
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/page1', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+
+		# Create the custom action model
+		custom_action_data = {'custom_action': CustomParams(text='test_value')}
+
+		class CustomActionModel(ActionModel):
+			custom_action: CustomParams | None = None
+
+		# Execute the custom action
+		result = await tools.act(CustomActionModel(**custom_action_data), browser_session)
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+		assert 'Custom action executed with: test_value on' in result.extracted_content
+		assert f'{base_url}/page1' in result.extracted_content
+
+	async def test_wait_action(self, tools, browser_session):
+		"""Test that the wait action correctly waits for the specified duration."""
+
+		# verify that it's in the default action set
+		wait_action = None
+		for action_name, action in tools.registry.registry.actions.items():
+			if 'wait' in action_name.lower() and 'seconds' in str(action.param_model.model_fields):
+				wait_action = action
+				break
+		assert wait_action is not None, 'Could not find wait action in tools'
+
+		# Check that it has seconds parameter with default
+		assert 'seconds' in wait_action.param_model.model_fields
+		schema = wait_action.param_model.model_json_schema()
+		assert schema['properties']['seconds']['default'] == 3
+
+		# Create wait action for 1 second - fix to use a dictionary
+		wait_action = {'wait': {'seconds': 3}}  # Corrected format
+
+		class WaitActionModel(ActionModel):
+			wait: dict | None = None
+
+		# Record start time
+		start_time = time.time()
+
+		# Execute wait action
+		result = await tools.act(WaitActionModel(**wait_action), browser_session)
+
+		# Record end time
+		end_time = time.time()
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+		assert 'Waited for' in result.extracted_content or 'Waiting for' in result.extracted_content
+
+		# Verify that approximately 1 second has passed (allowing some margin)
+		assert end_time - start_time <= 0.5  # We wait 3-3 seconds for LLM call
+
+		# longer wait
+		# Create wait action for 1 second - fix to use a dictionary
+		wait_action = {'wait': {'seconds': 5}}  # Corrected format
+
+		# Record start time
+		start_time = time.time()
+
+		# Execute wait action
+		result = await tools.act(WaitActionModel(**wait_action), browser_session)
+
+		# Record end time
+		end_time = time.time()
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+		assert 'Waited for' in result.extracted_content or 'Waiting for' in result.extracted_content
+
+		assert 1.5 <= end_time - start_time <= 2.5  # We wait 5-3 seconds for LLM call
+
+	async def test_go_back_action(self, tools, browser_session, base_url):
+		"""Test that go_back action navigates to the previous page."""
+		# Navigate to first page
+		goto_action1 = {'go_to_url': GoToUrlAction(url=f'{base_url}/page1', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action1), browser_session)
+
+		# Store the first page URL
+		first_url = await browser_session.get_current_page_url()
+		print(f'First page URL: {first_url}')
+
+		# Navigate to second page
+		goto_action2 = {'go_to_url': GoToUrlAction(url=f'{base_url}/page2', new_tab=False)}
+		await tools.act(GoToUrlActionModel(**goto_action2), browser_session)
+
+		# Verify we're on the second page
+		second_url = await browser_session.get_current_page_url()
+		print(f'Second page URL: {second_url}')
+		assert f'{base_url}/page2' in second_url
+
+		# Execute go back action
+		go_back_action = {'go_back': NoParamsAction()}
+
+		class GoBackActionModel(ActionModel):
+			go_back: NoParamsAction | None = None
+
+		result = await tools.act(GoBackActionModel(**go_back_action), browser_session)
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+		assert 'Navigated back' in result.extracted_content
+
+		# Add another delay to allow the navigation to complete
+		await asyncio.sleep(1)
+
+		# Verify we're back on a different page than before
+		final_url = await browser_session.get_current_page_url()
+		print(f'Final page URL after going back: {final_url}')
+
+		# Try to verify we're back on the first page, but don't fail the test if not
+		assert f'{base_url}/page1' in final_url, f'Expected to return to page1 but got {final_url}'
+
+	async def test_navigation_chain(self, tools, browser_session, base_url):
+		"""Test navigating through multiple pages and back through history."""
+		# Set up a chain of navigation: Home -> Page1 -> Page2
+		urls = [f'{base_url}/', f'{base_url}/page1', f'{base_url}/page2']
+
+		# Navigate to each page in sequence
+		for url in urls:
+			action_data = {'go_to_url': GoToUrlAction(url=url, new_tab=False)}
+
+			class GoToUrlActionModel(ActionModel):
+				go_to_url: GoToUrlAction | None = None
+
+			await tools.act(GoToUrlActionModel(**action_data), browser_session)
+
+			# Verify current page
+			current_url = await browser_session.get_current_page_url()
+			assert url in current_url
+
+		# Go back twice and verify each step
+		for expected_url in reversed(urls[:-1]):
+			go_back_action = {'go_back': NoParamsAction()}
+
+			class GoBackActionModel(ActionModel):
+				go_back: NoParamsAction | None = None
+
+			await tools.act(GoBackActionModel(**go_back_action), browser_session)
+			await asyncio.sleep(1)  # Wait for navigation to complete
+
+			current_url = await browser_session.get_current_page_url()
+			assert expected_url in current_url
+
+	async def test_excluded_actions(self, browser_session):
+		"""Test that excluded actions are not registered."""
+		# Create tools with excluded actions
+		excluded_tools = Tools(exclude_actions=['search_google', 'scroll'])
+
+		# Verify excluded actions are not in the registry
+		assert 'search_google' not in excluded_tools.registry.registry.actions
+		assert 'scroll' not in excluded_tools.registry.registry.actions
+
+		# But other actions are still there
+		assert 'go_to_url' in excluded_tools.registry.registry.actions
+		assert 'click_element_by_index' in excluded_tools.registry.registry.actions
+
+	async def test_search_google_action(self, tools, browser_session, base_url):
+		"""Test the search_google action."""
+
+		await browser_session.get_current_page_url()
+
+		# Execute search_google action - it will actually navigate to our search results page
+		search_action = {'search_google': SearchGoogleAction(query='Python web automation')}
+
+		class SearchGoogleActionModel(ActionModel):
+			search_google: SearchGoogleAction | None = None
+
+		result = await tools.act(SearchGoogleActionModel(**search_action), browser_session)
+
+		# Verify the result
+		assert isinstance(result, ActionResult)
+		assert result.extracted_content is not None
+		assert 'Searched' in result.extracted_content and 'Python web automation' in result.extracted_content
+
+		# For our test purposes, we just verify we're on some URL
+		current_url = await browser_session.get_current_page_url()
+		assert current_url is not None and 'Python' in current_url
+
+	async def test_done_action(self, tools, browser_session, base_url):
+		"""Test that DoneAction completes a task and reports success or failure."""
+		# Create a temporary directory for the file system
+		with tempfile.TemporaryDirectory() as temp_dir:
+			file_system = FileSystem(temp_dir)
+
+			# First navigate to a page
+			goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/page1', new_tab=False)}
+
+			class GoToUrlActionModel(ActionModel):
+				go_to_url: GoToUrlAction | None = None
+
+			await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+
+			success_done_message = 'Successfully completed task'
+
+			# Create done action with success
+			done_action = {'done': DoneAction(text=success_done_message, success=True)}
+
+			class DoneActionModel(ActionModel):
+				done: DoneAction | None = None
+
+			# Execute done action with file_system
+			result = await tools.act(DoneActionModel(**done_action), browser_session, file_system=file_system)
+
+			# Verify the result
+			assert isinstance(result, ActionResult)
+			assert result.extracted_content is not None
+			assert success_done_message in result.extracted_content
+			assert result.success is True
+			assert result.is_done is True
+			assert result.error is None
+
+			failed_done_message = 'Failed to complete task'
+
+			# Test with failure case
+			failed_done_action = {'done': DoneAction(text=failed_done_message, success=False)}
+
+			# Execute failed done action with file_system
+			result = await tools.act(DoneActionModel(**failed_done_action), browser_session, file_system=file_system)
+
+			# Verify the result
+			assert isinstance(result, ActionResult)
+			assert result.extracted_content is not None
+			assert failed_done_message in result.extracted_content
+			assert result.success is False
+			assert result.is_done is True
+			assert result.error is None
+
+	async def test_get_dropdown_options(self, tools, browser_session, base_url, http_server):
+		"""Test that get_dropdown_options correctly retrieves options from a dropdown."""
+		# Add route for dropdown test page
+		http_server.expect_request('/dropdown1').respond_with_data(
+			"""
+			<!DOCTYPE html>
+			<html>
+			<head>
+				<title>Dropdown Test</title>
+			</head>
+			<body>
+				<h1>Dropdown Test</h1>
+				<select id="test-dropdown" name="test-dropdown">
+					<option value="">Please select</option>
+					<option value="option1">First Option</option>
+					<option value="option2">Second Option</option>
+					<option value="option3">Third Option</option>
+				</select>
+			</body>
+			</html>
+			""",
+			content_type='text/html',
+		)
+
+		# Navigate to the dropdown test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/dropdown1', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+
+		# Wait for the page to load using CDP
+		cdp_session = browser_session.agent_focus
+		assert cdp_session is not None, 'CDP session not initialized'
+
+		# Wait for page load by checking document ready state
+		await asyncio.sleep(0.5)  # Brief wait for navigation to start
+		ready_state = await cdp_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': 'document.readyState'}, session_id=cdp_session.session_id
+		)
+		# If not complete, wait a bit more
+		if ready_state.get('result', {}).get('value') != 'complete':
+			await asyncio.sleep(1.0)
+
+		# Initialize the DOM state to populate the selector map
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Get the selector map
+		selector_map = await browser_session.get_selector_map()
+
+		# Find the dropdown element in the selector map
+		dropdown_index = None
+		for idx, element in selector_map.items():
+			if element.tag_name.lower() == 'select':
+				dropdown_index = idx
+				break
+
+		assert dropdown_index is not None, (
+			f'Could not find select element in selector map. Available elements: {[f"{idx}: {element.tag_name}" for idx, element in selector_map.items()]}'
+		)
+
+		# Create a model for the standard get_dropdown_options action
+		class GetDropdownOptionsModel(ActionModel):
+			get_dropdown_options: dict[str, int]
+
+		# Execute the action with the dropdown index
+		result = await tools.act(
+			action=GetDropdownOptionsModel(get_dropdown_options={'index': dropdown_index}),
+			browser_session=browser_session,
+		)
+
+		expected_options = [
+			{'index': 0, 'text': 'Please select', 'value': ''},
+			{'index': 1, 'text': 'First Option', 'value': 'option1'},
+			{'index': 2, 'text': 'Second Option', 'value': 'option2'},
+			{'index': 3, 'text': 'Third Option', 'value': 'option3'},
+		]
+
+		# Verify the result structure
+		assert isinstance(result, ActionResult)
+
+		# Core logic validation: Verify all options are returned
+		assert result.extracted_content is not None
+		for option in expected_options[1:]:  # Skip the placeholder option
+			assert option['text'] in result.extracted_content, f"Option '{option['text']}' not found in result content"
+
+		# Verify the instruction for using the text in select_dropdown_option is included
+		assert (
+			'Use the exact text or value string' in result.extracted_content
+			and 'select_dropdown_option' in result.extracted_content
+		)
+
+		# Verify the actual dropdown options in the DOM using CDP
+		dropdown_options_result = await cdp_session.cdp_client.send.Runtime.evaluate(
+			params={
+				'expression': """
+					JSON.stringify((() => {
+						const select = document.getElementById('test-dropdown');
+						return Array.from(select.options).map(opt => ({
+							text: opt.text,
+							value: opt.value
+						}));
+					})())
+				""",
+				'returnByValue': True,
+			},
+			session_id=cdp_session.session_id,
+		)
+		dropdown_options_json = dropdown_options_result.get('result', {}).get('value', '[]')
+		import json
+
+		dropdown_options = json.loads(dropdown_options_json) if isinstance(dropdown_options_json, str) else dropdown_options_json
+
+		# Verify the dropdown has the expected options
+		assert len(dropdown_options) == len(expected_options), (
+			f'Expected {len(expected_options)} options, got {len(dropdown_options)}'
+		)
+		for i, expected in enumerate(expected_options):
+			actual = dropdown_options[i]
+			assert actual['text'] == expected['text'], (
+				f"Option at index {i} has wrong text: expected '{expected['text']}', got '{actual['text']}'"
+			)
+			assert actual['value'] == expected['value'], (
+				f"Option at index {i} has wrong value: expected '{expected['value']}', got '{actual['value']}'"
+			)
+
+	async def test_select_dropdown_option(self, tools, browser_session, base_url, http_server):
+		"""Test that select_dropdown_option correctly selects an option from a dropdown."""
+		# Add route for dropdown test page
+		http_server.expect_request('/dropdown2').respond_with_data(
+			"""
+			<!DOCTYPE html>
+			<html>
+			<head>
+				<title>Dropdown Test</title>
+			</head>
+			<body>
+				<h1>Dropdown Test</h1>
+				<select id="test-dropdown" name="test-dropdown">
+					<option value="">Please select</option>
+					<option value="option1">First Option</option>
+					<option value="option2">Second Option</option>
+					<option value="option3">Third Option</option>
+				</select>
+			</body>
+			</html>
+			""",
+			content_type='text/html',
+		)
+
+		# Navigate to the dropdown test page
+		goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/dropdown2', new_tab=False)}
+
+		class GoToUrlActionModel(ActionModel):
+			go_to_url: GoToUrlAction | None = None
+
+		await tools.act(GoToUrlActionModel(**goto_action), browser_session)
+
+		# Wait for the page to load using CDP
+		cdp_session = browser_session.agent_focus
+		assert cdp_session is not None, 'CDP session not initialized'
+
+		# Wait for page load by checking document ready state
+		await asyncio.sleep(0.5)  # Brief wait for navigation to start
+		ready_state = await cdp_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': 'document.readyState'}, session_id=cdp_session.session_id
+		)
+		# If not complete, wait a bit more
+		if ready_state.get('result', {}).get('value') != 'complete':
+			await asyncio.sleep(1.0)
+
+		# populate the selector map with highlight indices
+		await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
+
+		# Now get the selector map which should contain our dropdown
+		selector_map = await browser_session.get_selector_map()
+
+		# Find the dropdown element in the selector map
+		dropdown_index = None
+		for idx, element in selector_map.items():
+			if element.tag_name.lower() == 'select':
+				dropdown_index = idx
+				break
+
+		assert dropdown_index is not None, (
+			f'Could not find select element in selector map. Available elements: {[f"{idx}: {element.tag_name}" for idx, element in selector_map.items()]}'
+		)
+
+		# Create a model for the standard select_dropdown_option action
+		class SelectDropdownOptionModel(ActionModel):
+			select_dropdown_option: dict
+
+		# Execute the action with the dropdown index
+		result = await tools.act(
+			SelectDropdownOptionModel(select_dropdown_option={'index': dropdown_index, 'text': 'Second Option'}),
+			browser_session,
+		)
+
+		# Verify the result structure
+		assert isinstance(result, ActionResult)
+
+		# Core logic validation: Verify selection was successful
+		assert result.extracted_content is not None
+		assert 'selected option' in result.extracted_content.lower()
+		assert 'Second Option' in result.extracted_content
+
+		# Verify the actual dropdown selection was made by checking the DOM using CDP
+		selected_value_result = await cdp_session.cdp_client.send.Runtime.evaluate(
+			params={'expression': "document.getElementById('test-dropdown').value"}, session_id=cdp_session.session_id
+		)
+		selected_value = selected_value_result.get('result', {}).get('value')
+		assert selected_value == 'option2'  # Second Option has value "option2"

chapter9/browser-use-rpa/browser-use/tests/ci/test_url_shortening.py

@@ -0,0 +1,161 @@
+"""
+Simplified tests for URL shortening functionality in Agent service.
+
+Three focused tests:
+1. Input message processing with URL shortening
+2. Output processing with custom actions and URL restoration
+3. End-to-end pipeline test
+"""
+
+import json
+
+import pytest
+
+from browser_use.agent.service import Agent
+from browser_use.agent.views import AgentOutput
+from browser_use.llm.messages import AssistantMessage, BaseMessage, UserMessage
+
+# Super long URL to reuse across tests - much longer than the 25 character limit
+# Includes both query params (?...) and fragment params (#...)
+SUPER_LONG_URL = 'https://documentation.example-company.com/api/v3/enterprise/user-management/endpoints/administration/create-new-user-account-with-permissions/advanced-settings?format=detailed-json&version=3.2.1&timestamp=1699123456789&session_id=abc123def456ghi789&authentication_token=very_long_authentication_token_string_here&include_metadata=true&expand_relationships=user_groups,permissions,roles&sort_by=created_at&order=desc&page_size=100&include_deprecated_fields=false&api_key=super_long_api_key_that_exceeds_normal_limits#section=user_management&tab=advanced&view=detailed&scroll_to=permissions_table&highlight=admin_settings&filter=active_users&expand_all=true&debug_mode=enabled'
+
+
+@pytest.fixture
+def agent():
+	"""Create an agent instance for testing URL shortening functionality."""
+	from tests.ci.conftest import create_mock_llm
+
+	return Agent(task='Test URL shortening', llm=create_mock_llm(), url_shortening_limit=25)
+
+
+class TestUrlShorteningInputProcessing:
+	"""Test URL shortening for input messages."""
+
+	def test_process_input_messages_with_url_shortening(self, agent: Agent):
+		"""Test that long URLs in input messages are shortened and mappings stored."""
+		original_content = f'Please visit {SUPER_LONG_URL} and extract information'
+
+		messages: list[BaseMessage] = [UserMessage(content=original_content)]
+
+		# Process messages (modifies messages in-place and returns URL mappings)
+		url_mappings = agent._process_messsages_and_replace_long_urls_shorter_ones(messages)
+
+		# Verify URL was shortened in the message (modified in-place)
+		processed_content = messages[0].content or ''
+		assert processed_content != original_content
+		assert 'https://documentation.example-company.com' in processed_content
+		assert len(processed_content) < len(original_content)
+
+		# Verify URL mapping was returned
+		assert len(url_mappings) == 1
+		shortened_url = next(iter(url_mappings.keys()))
+		assert url_mappings[shortened_url] == SUPER_LONG_URL
+
+	def test_process_user_and_assistant_messages_with_url_shortening(self, agent: Agent):
+		"""Test URL shortening in both UserMessage and AssistantMessage."""
+		user_content = f'I need to access {SUPER_LONG_URL} for the API documentation'
+		assistant_content = f'I will help you navigate to {SUPER_LONG_URL} to retrieve the documentation'
+
+		messages: list[BaseMessage] = [UserMessage(content=user_content), AssistantMessage(content=assistant_content)]
+
+		# Process messages (modifies messages in-place and returns URL mappings)
+		url_mappings = agent._process_messsages_and_replace_long_urls_shorter_ones(messages)
+
+		# Verify URL was shortened in both messages
+		user_processed_content = messages[0].content or ''
+		assistant_processed_content = messages[1].content or ''
+
+		assert user_processed_content != user_content
+		assert assistant_processed_content != assistant_content
+		assert 'https://documentation.example-company.com' in user_processed_content
+		assert 'https://documentation.example-company.com' in assistant_processed_content
+		assert len(user_processed_content) < len(user_content)
+		assert len(assistant_processed_content) < len(assistant_content)
+
+		# Verify URL mapping was returned (should be same shortened URL for both occurrences)
+		assert len(url_mappings) == 1
+		shortened_url = next(iter(url_mappings.keys()))
+		assert url_mappings[shortened_url] == SUPER_LONG_URL
+
+
+class TestUrlShorteningOutputProcessing:
+	"""Test URL restoration for output processing with custom actions."""
+
+	def test_process_output_with_custom_actions_and_url_restoration(self, agent: Agent):
+		"""Test that shortened URLs in AgentOutput with custom actions are restored."""
+		# Set up URL mapping (simulating previous shortening)
+		shortened_url: str = agent._replace_urls_in_text(SUPER_LONG_URL)[0]
+		url_mappings = {shortened_url: SUPER_LONG_URL}
+
+		# Create AgentOutput with shortened URLs using JSON parsing
+		output_json = {
+			'thinking': f'I need to navigate to {shortened_url} for documentation',
+			'evaluation_previous_goal': 'Successfully processed the request',
+			'memory': f'Found useful info at {shortened_url}',
+			'next_goal': 'Complete the documentation review',
+			'action': [{'go_to_url': {'url': shortened_url, 'new_tab': False}}],
+		}
+
+		# Create properly typed AgentOutput with custom actions
+		tools = agent.tools
+		ActionModel = tools.registry.create_action_model()
+		AgentOutputWithActions = AgentOutput.type_with_custom_actions(ActionModel)
+		agent_output = AgentOutputWithActions.model_validate_json(json.dumps(output_json))
+
+		# Process the output to restore URLs (modifies agent_output in-place)
+		agent._recursive_process_all_strings_inside_pydantic_model(agent_output, url_mappings)
+
+		# Verify URLs were restored in all locations
+		assert SUPER_LONG_URL in (agent_output.thinking or '')
+		assert SUPER_LONG_URL in (agent_output.memory or '')
+		action_data = agent_output.action[0].model_dump()
+		assert action_data['go_to_url']['url'] == SUPER_LONG_URL
+
+
+class TestUrlShorteningEndToEnd:
+	"""Test complete URL shortening pipeline end-to-end."""
+
+	def test_complete_url_shortening_pipeline(self, agent: Agent):
+		"""Test the complete pipeline: input shortening -> processing -> output restoration."""
+
+		# Step 1: Input processing with URL shortening
+		original_content = f'Navigate to {SUPER_LONG_URL} and extract the API documentation'
+
+		messages: list[BaseMessage] = [UserMessage(content=original_content)]
+
+		url_mappings = agent._process_messsages_and_replace_long_urls_shorter_ones(messages)
+
+		# Verify URL was shortened in input
+		assert len(url_mappings) == 1
+		shortened_url = next(iter(url_mappings.keys()))
+		assert url_mappings[shortened_url] == SUPER_LONG_URL
+		assert shortened_url in (messages[0].content or '')
+
+		# Step 2: Simulate agent output with shortened URL
+		output_json = {
+			'thinking': f'I will navigate to {shortened_url} to get the documentation',
+			'evaluation_previous_goal': 'Starting documentation extraction',
+			'memory': f'Target URL: {shortened_url}',
+			'next_goal': 'Extract API documentation',
+			'action': [{'go_to_url': {'url': shortened_url, 'new_tab': True}}],
+		}
+
+		# Create AgentOutput with custom actions
+		tools = agent.tools
+		ActionModel = tools.registry.create_action_model()
+		AgentOutputWithActions = AgentOutput.type_with_custom_actions(ActionModel)
+		agent_output = AgentOutputWithActions.model_validate_json(json.dumps(output_json))
+
+		# Step 3: Output processing with URL restoration (modifies agent_output in-place)
+		agent._recursive_process_all_strings_inside_pydantic_model(agent_output, url_mappings)
+
+		# Verify complete pipeline worked correctly
+		assert SUPER_LONG_URL in (agent_output.thinking or '')
+		assert SUPER_LONG_URL in (agent_output.memory or '')
+		action_data = agent_output.action[0].model_dump()
+		assert action_data['go_to_url']['url'] == SUPER_LONG_URL
+		assert action_data['go_to_url']['new_tab'] is True
+
+		# Verify original shortened content is no longer present
+		assert shortened_url not in (agent_output.thinking or '')
+		assert shortened_url not in (agent_output.memory or '')

chapter9/browser-use-rpa/browser-use/tests/scripts/debug_iframe_scrolling.py

@@ -0,0 +1,301 @@
+"""
+Debug test for iframe scrolling issue where DOM tree only shows top elements after scrolling.
+
+This test verifies that after scrolling inside an iframe, the selector_map correctly
+contains lower input elements like City, State, Zip Code, etc.
+"""
+
+import asyncio
+import sys
+from pathlib import Path
+
+# Add parent directory to path to import browser_use modules
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from browser_use.agent.service import Agent
+from browser_use.agent.views import ActionModel
+from browser_use.browser import BrowserProfile, BrowserSession
+from browser_use.browser.events import BrowserStateRequestEvent
+
+# Import the mock LLM helper from conftest
+from tests.ci.conftest import create_mock_llm
+
+
+async def debug_iframe_scrolling():
+	"""Debug iframe scrolling and DOM visibility issue."""
+
+	print('Starting iframe scrolling debug test...')
+
+	# Create the sequence of actions for the mock LLM
+	# We need to format these as the LLM would return them
+	actions = [
+		# First action: Navigate to the test URL
+		"""
+		{
+			"thinking": "Navigating to the iframe test page",
+			"evaluation_previous_goal": null,
+			"memory": "Starting test",
+			"next_goal": "Navigate to the iframe test page",
+			"action": [
+				{
+					"go_to_url": {
+						"url": "https://browser-use.github.io/stress-tests/challenges/iframe-inception-level1.html",
+						"new_tab": false
+					}
+				}
+			]
+		}
+		""",
+		# Second action: Input text in the first name field (to verify we can interact)
+		"""
+		{
+			"thinking": "Inputting text in the first name field to test interaction",
+			"evaluation_previous_goal": "Successfully navigated to the page",
+			"memory": "Page loaded with nested iframes",
+			"next_goal": "Type text in the first name field",
+			"action": [
+				{
+					"input_text": {
+						"index": 1,
+						"text": "TestName"
+					}
+				}
+			]
+		}
+		""",
+		# Third action: Scroll the iframe (element_index=2 should be the iframe)
+		"""
+		{
+			"thinking": "Scrolling inside the iframe to reveal lower form elements",
+			"evaluation_previous_goal": "Successfully typed in first name field",
+			"memory": "Typed TestName in first field",
+			"next_goal": "Scroll inside the innermost iframe to see more form fields",
+			"action": [
+				{
+					"scroll": {
+						"down": true,
+						"num_pages": 1.0,
+						"index": 2
+					}
+				}
+			]
+		}
+		""",
+		# Fourth action: Done
+		"""
+		{
+			"thinking": "Completed scrolling, ready to inspect DOM",
+			"evaluation_previous_goal": "Successfully scrolled inside iframe",
+			"memory": "Scrolled to reveal lower form fields",
+			"next_goal": "Task completed",
+			"action": [
+				{
+					"done": {
+						"text": "Scrolling completed",
+						"success": true
+					}
+				}
+			]
+		}
+		""",
+	]
+
+	# Create mock LLM with our action sequence
+	mock_llm = create_mock_llm(actions=actions)
+
+	# Create browser session with headless=False so we can see what's happening
+	browser_session = BrowserSession(
+		browser_profile=BrowserProfile(
+			headless=False,  # Set to False to see the browser
+			user_data_dir=None,  # Use temporary directory
+			keep_alive=True,
+			enable_default_extensions=True,
+			cross_origin_iframes=True,  # Enable cross-origin iframe support
+		)
+	)
+
+	try:
+		# Start the browser session
+		await browser_session.start()
+		print('Browser session started')
+
+		# Create an agent with the mock LLM
+		agent = Agent(
+			task='Navigate to the iframe test page and scroll inside the iframe',
+			llm=mock_llm,
+			browser_session=browser_session,
+		)
+
+		# Helper function to capture and analyze DOM state
+		async def capture_dom_state(label: str) -> dict:
+			"""Capture DOM state and return analysis"""
+			print(f'\n📸 Capturing DOM state: {label}')
+			state_event = browser_session.event_bus.dispatch(
+				BrowserStateRequestEvent(
+					include_dom=True, include_screenshot=False, cache_clickable_elements_hashes=True, include_recent_events=False
+				)
+			)
+			browser_state = await state_event.event_result()
+
+			if browser_state and browser_state.dom_state and browser_state.dom_state.selector_map:
+				selector_map = browser_state.dom_state.selector_map
+				element_count = len(selector_map)
+
+				# Check for specific elements
+				found_elements = {}
+				expected_checks = [
+					('First Name', ['firstName', 'first name']),
+					('Last Name', ['lastName', 'last name']),
+					('Email', ['email']),
+					('City', ['city']),
+					('State', ['state']),
+					('Zip', ['zip', 'zipCode']),
+				]
+
+				for name, keywords in expected_checks:
+					for index, element in selector_map.items():
+						element_str = str(element).lower()
+						if any(kw.lower() in element_str for kw in keywords):
+							found_elements[name] = True
+							break
+
+				return {
+					'label': label,
+					'total_elements': element_count,
+					'found_elements': found_elements,
+					'selector_map': selector_map,
+				}
+			return {'label': label, 'error': 'No DOM state available'}
+
+		# Capture initial state before any actions
+		print('\n' + '=' * 80)
+		print('PHASE 1: INITIAL PAGE LOAD')
+		print('=' * 80)
+
+		# Navigate to the page first
+		from browser_use.tools.service import Tools
+
+		tools = Tools()
+
+		# Create the action model for navigation
+		goto_action = ActionModel.model_validate_json(actions[0])
+		await tools.act(goto_action, browser_session)
+		await asyncio.sleep(2)  # Wait for page to fully load
+
+		initial_state = await capture_dom_state('INITIAL (after page load)')
+
+		# Now run the rest of the actions via the agent
+		print('\n' + '=' * 80)
+		print('PHASE 2: EXECUTING ACTIONS')
+		print('=' * 80)
+
+		# Create new agent with remaining actions
+		remaining_actions = actions[1:]  # Skip the navigation we already did
+		mock_llm_remaining = create_mock_llm(actions=remaining_actions)
+		agent = Agent(
+			task='Input text and scroll inside the iframe',
+			llm=mock_llm_remaining,
+			browser_session=browser_session,
+		)
+
+		# Hook into agent actions to capture state after each one
+		states = []
+		original_act = tools.act
+
+		async def wrapped_act(action, session):
+			result = await original_act(action, session)
+			# Capture state after each action
+			action_type = 'unknown'
+			if hasattr(action, 'input_text') and action.input_text:
+				action_type = 'input_text'
+				await asyncio.sleep(1)  # Give time for DOM to update
+				state = await capture_dom_state('AFTER INPUT_TEXT')
+				states.append(state)
+			elif hasattr(action, 'scroll') and action.scroll:
+				action_type = 'scroll'
+				await asyncio.sleep(2)  # Give more time after scroll
+				state = await capture_dom_state('AFTER SCROLL')
+				states.append(state)
+			return result
+
+		tools.act = wrapped_act
+
+		# Run the agent with remaining actions
+		result = await agent.run()
+		print(f'\nAgent completed with result: {result}')
+
+		# Analyze all captured states
+		print('\n' + '=' * 80)
+		print('PHASE 3: ANALYSIS OF DOM STATES')
+		print('=' * 80)
+
+		all_states = [initial_state] + states
+
+		for state in all_states:
+			if 'error' in state:
+				print(f'\n❌ {state["label"]}: {state["error"]}')
+			else:
+				print(f'\n📊 {state["label"]}:')
+				print(f'  Total elements: {state["total_elements"]}')
+				print('  Found elements:')
+				for elem_name, found in state['found_elements'].items():
+					status = '✓' if found else '✗'
+					print(f'    {status} {elem_name}')
+
+		# Compare states
+		print('\n' + '=' * 80)
+		print('COMPARISON SUMMARY')
+		print('=' * 80)
+
+		if len(all_states) >= 3:
+			initial = all_states[0]
+			after_input = all_states[1] if len(all_states) > 1 else None
+			after_scroll = all_states[2] if len(all_states) > 2 else None
+
+			print('\nElement count changes:')
+			print(f'  Initial: {initial.get("total_elements", 0)} elements')
+			if after_input:
+				print(f'  After input_text: {after_input.get("total_elements", 0)} elements')
+			if after_scroll:
+				print(f'  After scroll: {after_scroll.get("total_elements", 0)} elements')
+
+			# Check if lower form fields appear after scroll
+			if after_scroll and 'found_elements' in after_scroll:
+				lower_fields = ['City', 'State', 'Zip']
+				missing_fields = [f for f in lower_fields if not after_scroll['found_elements'].get(f, False)]
+
+				if missing_fields:
+					print('\n⚠️  BUG CONFIRMED: Lower form fields missing after scroll:')
+					for field in missing_fields:
+						print(f'    ✗ {field}')
+					print('\nThis confirms that scrolling inside iframes does not update the DOM tree properly.')
+				else:
+					print('\n✅ SUCCESS: All lower form fields are visible after scrolling!')
+
+		# Show first few elements from final state for debugging
+		if states and 'selector_map' in states[-1]:
+			print('\n' + '=' * 80)
+			print('DEBUG: First 5 elements in final selector_map')
+			print('=' * 80)
+			final_map = states[-1]['selector_map']
+			for i, (index, element) in enumerate(list(final_map.items())[:5]):
+				elem_preview = str(element)[:150]
+				print(f'\n  [{index}]: {elem_preview}...')
+
+		# Keep browser open for manual inspection if needed
+		print('\n' + '=' * 80)
+		print('Test complete. Browser will remain open for 10 seconds for inspection...')
+		print('=' * 80)
+		await asyncio.sleep(10)
+
+	finally:
+		# Clean up
+		print('\nCleaning up...')
+		await browser_session.kill()
+		await browser_session.event_bus.stop(clear=True, timeout=5)
+		print('Browser session closed')
+
+
+if __name__ == '__main__':
+	# Run the debug test
+	asyncio.run(debug_iframe_scrolling())

chapter9/browser-use-rpa/browser-use/tests/scripts/test_frame_hierarchy.py

@@ -0,0 +1,258 @@
+#!/usr/bin/env python3
+"""Test frame hierarchy for any URL passed as argument."""
+
+import asyncio
+import sys
+
+from browser_use.browser import BrowserSession
+from browser_use.browser.events import BrowserStartEvent
+from browser_use.browser.profile import BrowserProfile
+
+
+async def analyze_frame_hierarchy(url):
+	"""Analyze and display complete frame hierarchy for a URL."""
+
+	profile = BrowserProfile(headless=True, user_data_dir=None)
+	session = BrowserSession(browser_profile=profile)
+
+	try:
+		print('🚀 Starting browser...')
+		await session.on_BrowserStartEvent(BrowserStartEvent())
+
+		print(f'📍 Navigating to: {url}')
+		await session._cdp_navigate(url)
+		await asyncio.sleep(3)
+
+		print('\n' + '=' * 80)
+		print('FRAME HIERARCHY ANALYSIS')
+		print('=' * 80)
+
+		# Get all targets
+		targets = await session.cdp_client.send.Target.getTargets()
+		all_targets = targets.get('targetInfos', [])
+
+		# Separate by type
+		page_targets = [t for t in all_targets if t.get('type') == 'page']
+		iframe_targets = [t for t in all_targets if t.get('type') == 'iframe']
+
+		print('\n📊 Target Summary:')
+		print(f'  Total targets: {len(all_targets)}')
+		print(f'  Page targets: {len(page_targets)}')
+		print(f'  Iframe targets (OOPIFs): {len(iframe_targets)}')
+
+		# Show all targets
+		print('\n📋 All Targets:')
+		for i, target in enumerate(all_targets):
+			t_type = target.get('type')
+			t_url = target.get('url', 'none')
+			t_id = target.get('targetId', 'unknown')
+
+			if t_type in ['page', 'iframe']:
+				print(f'\n  [{i + 1}] Type: {t_type}')
+				print(f'      URL: {t_url}')
+				print(f'      Target ID: {t_id[:30]}...')
+				print(f'      Attached: {target.get("attached", False)}')
+
+		# Get main page frame tree
+		main_target = next((t for t in page_targets if url in t.get('url', '')), page_targets[0] if page_targets else None)
+
+		if main_target:
+			print('\n📐 Main Page Frame Tree:')
+			print(f'  Target: {main_target["url"]}')
+			print(f'  Target ID: {main_target["targetId"][:30]}...')
+
+			s = await session.cdp_client.send.Target.attachToTarget(params={'targetId': main_target['targetId'], 'flatten': True})
+			sid = s['sessionId']
+
+			try:
+				await session.cdp_client.send.Page.enable(session_id=sid)
+				tree = await session.cdp_client.send.Page.getFrameTree(session_id=sid)
+
+				print('\n  Frame Tree Structure:')
+
+				def print_tree(node, indent=0, parent_id=None):
+					frame = node['frame']
+					frame_id = frame.get('id', 'unknown')
+					frame_url = frame.get('url', 'none')
+
+					prefix = '  ' * indent + ('└─ ' if indent > 0 else '')
+					print(f'{prefix}Frame: {frame_url}')
+					print(f'{"  " * (indent + 1)}ID: {frame_id[:30]}...')
+
+					if parent_id:
+						print(f'{"  " * (indent + 1)}Parent: {parent_id[:30]}...')
+
+					# Check cross-origin status
+					cross_origin = frame.get('crossOriginIsolatedContextType', 'unknown')
+					if cross_origin != 'NotIsolated':
+						print(f'{"  " * (indent + 1)}⚠️  Cross-Origin: {cross_origin}')
+
+					# Process children
+					for child in node.get('childFrames', []):
+						print_tree(child, indent + 1, frame_id)
+
+				print_tree(tree['frameTree'])
+
+			finally:
+				await session.cdp_client.send.Target.detachFromTarget(params={'sessionId': sid})
+
+		# Show iframe target trees
+		if iframe_targets:
+			print('\n🔸 OOPIF Target Frame Trees:')
+
+			for iframe_target in iframe_targets:
+				print(f'\n  OOPIF Target: {iframe_target["url"]}')
+				print(f'  Target ID: {iframe_target["targetId"][:30]}...')
+
+				s = await session.cdp_client.send.Target.attachToTarget(
+					params={'targetId': iframe_target['targetId'], 'flatten': True}
+				)
+				sid = s['sessionId']
+
+				try:
+					await session.cdp_client.send.Page.enable(session_id=sid)
+					tree = await session.cdp_client.send.Page.getFrameTree(session_id=sid)
+
+					frame = tree['frameTree']['frame']
+					print(f'  Frame ID: {frame.get("id", "unknown")[:30]}...')
+					print(f'  Frame URL: {frame.get("url", "none")}')
+					print('  ⚠️  This frame runs in a separate process (OOPIF)')
+
+				except Exception as e:
+					print(f'  Error: {e}')
+				finally:
+					await session.cdp_client.send.Target.detachFromTarget(params={'sessionId': sid})
+
+		# Now show unified view from get_all_frames
+		print('\n' + '=' * 80)
+		print('UNIFIED FRAME HIERARCHY (get_all_frames method)')
+		print('=' * 80)
+
+		all_frames, target_sessions = await session.get_all_frames()
+
+		# Clean up sessions
+		for tid, sess_id in target_sessions.items():
+			try:
+				await session.cdp_client.send.Target.detachFromTarget(params={'sessionId': sess_id})
+			except Exception:
+				pass
+
+		print('\n📊 Frame Statistics:')
+		print(f'  Total frames discovered: {len(all_frames)}')
+
+		# Separate root and child frames
+		root_frames = []
+		child_frames = []
+
+		for frame_id, frame_info in all_frames.items():
+			if not frame_info.get('parentFrameId'):
+				root_frames.append((frame_id, frame_info))
+			else:
+				child_frames.append((frame_id, frame_info))
+
+		print(f'  Root frames: {len(root_frames)}')
+		print(f'  Child frames: {len(child_frames)}')
+
+		# Display all frames with details
+		print('\n📋 All Frames:')
+
+		for i, (frame_id, frame_info) in enumerate(all_frames.items()):
+			url = frame_info.get('url', 'none')
+			parent = frame_info.get('parentFrameId')
+			target_id = frame_info.get('frameTargetId', 'unknown')
+			is_cross = frame_info.get('isCrossOrigin', False)
+
+			print(f'\n  [{i + 1}] Frame URL: {url}')
+			print(f'      Frame ID: {frame_id[:30]}...')
+			print(f'      Parent Frame ID: {parent[:30] + "..." if parent else "None (ROOT)"}')
+			print(f'      Target ID: {target_id[:30]}...')
+			print(f'      Cross-Origin: {is_cross}')
+
+			# Highlight problems
+			if not parent and 'v0-simple-landing' in url:
+				print('      ❌ PROBLEM: Cross-origin frame incorrectly marked as root!')
+			elif not parent and url != 'about:blank' and url not in ['chrome://newtab/', 'about:blank']:
+				# Check if this should be the main frame
+				if any(url in t.get('url', '') for t in page_targets):
+					print('      ✅ Correctly identified as root frame')
+
+			if is_cross:
+				print('      🔸 This is a cross-origin frame (OOPIF)')
+
+		# Show parent-child relationships
+		print('\n🌳 Frame Relationships:')
+
+		# Build a tree structure
+		def print_frame_tree(frame_id, frame_info, indent=0, visited=None):
+			if visited is None:
+				visited = set()
+
+			if frame_id in visited:
+				return
+			visited.add(frame_id)
+
+			url = frame_info.get('url', 'none')
+			prefix = '  ' * indent + ('└─ ' if indent > 0 else '')
+
+			print(f'{prefix}{url[:60]}...')
+			print(f'{"  " * (indent + 1)}[{frame_id[:20]}...]')
+
+			# Find children
+			for child_id, child_info in all_frames.items():
+				if child_info.get('parentFrameId') == frame_id:
+					print_frame_tree(child_id, child_info, indent + 1, visited)
+
+		# Print trees starting from roots
+		for frame_id, frame_info in root_frames:
+			print('\n  Tree starting from root:')
+			print_frame_tree(frame_id, frame_info)
+
+		print('\n' + '=' * 80)
+		print('✅ Analysis complete!')
+		print('=' * 80)
+
+	except Exception as e:
+		print(f'❌ Error: {e}')
+		import traceback
+
+		traceback.print_exc()
+	finally:
+		# Stop the CDP client first before killing the browser
+		print('\n🛑 Shutting down...')
+
+		# Close CDP connection first while browser is still alive
+		if session._cdp_client_root:
+			try:
+				await session._cdp_client_root.stop()
+			except Exception:
+				pass  # Ignore errors if already disconnected
+
+		# Then stop the browser process
+		from browser_use.browser.events import BrowserStopEvent
+
+		stop_event = session.event_bus.dispatch(BrowserStopEvent())
+		try:
+			await asyncio.wait_for(stop_event, timeout=2.0)
+		except TimeoutError:
+			print('⚠️ Browser stop timed out')
+
+
+def main():
+	if len(sys.argv) != 2:
+		print('Usage: python test_frame_hierarchy.py <URL>')
+		print('\nExample URLs to test:')
+		print('  https://v0-website-with-clickable-elements.vercel.app/nested-iframe')
+		print('  https://v0-website-with-clickable-elements.vercel.app/cross-origin')
+		print('  https://v0-website-with-clickable-elements.vercel.app/shadow-dom')
+		sys.exit(1)
+
+	url = sys.argv[1]
+	asyncio.run(analyze_frame_hierarchy(url))
+
+	# Ensure clean exit
+	print('✅ Script completed')
+	sys.exit(0)
+
+
+if __name__ == '__main__':
+	main()