ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter9/browser-use-rpa/browser-use/browser_use/llm/tests/test_anthropic_cache.py

@@ -0,0 +1,290 @@
+import logging
+from typing import cast
+
+from browser_use.agent.service import Agent
+from browser_use.llm.anthropic.chat import ChatAnthropic
+from browser_use.llm.anthropic.serializer import AnthropicMessageSerializer, NonSystemMessage
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	Function,
+	ImageURL,
+	SystemMessage,
+	ToolCall,
+	UserMessage,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class TestAnthropicCache:
+	"""Comprehensive test for Anthropic cache serialization."""
+
+	def test_cache_basic_functionality(self):
+		"""Test basic cache functionality for all message types."""
+		# Test cache with different message types
+		messages: list[BaseMessage] = [
+			SystemMessage(content='System message!', cache=True),
+			UserMessage(content='User message!', cache=True),
+			AssistantMessage(content='Assistant message!', cache=False),
+		]
+
+		anthropic_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages)
+
+		assert len(anthropic_messages) == 2
+		assert isinstance(system_message, list)
+		assert isinstance(anthropic_messages[0]['content'], list)
+		assert isinstance(anthropic_messages[1]['content'], str)
+
+		# Test cache with assistant message
+		agent_messages: list[BaseMessage] = [
+			SystemMessage(content='System message!'),
+			UserMessage(content='User message!'),
+			AssistantMessage(content='Assistant message!', cache=True),
+		]
+
+		anthropic_messages, system_message = AnthropicMessageSerializer.serialize_messages(agent_messages)
+
+		assert isinstance(system_message, str)
+		assert isinstance(anthropic_messages[0]['content'], str)
+		assert isinstance(anthropic_messages[1]['content'], list)
+
+	def test_cache_with_tool_calls(self):
+		"""Test cache functionality with tool calls."""
+		tool_call = ToolCall(id='test_id', function=Function(name='test_function', arguments='{"arg": "value"}'))
+
+		# Assistant with tool calls and cache
+		assistant_with_tools = AssistantMessage(content='Assistant with tools', tool_calls=[tool_call], cache=True)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([assistant_with_tools])
+
+		assert len(messages) == 1
+		assert isinstance(messages[0]['content'], list)
+		# Should have both text and tool_use blocks
+		assert len(messages[0]['content']) >= 2
+
+	def test_cache_with_images(self):
+		"""Test cache functionality with image content."""
+		user_with_image = UserMessage(
+			content=[
+				ContentPartTextParam(text='Here is an image:', type='text'),
+				ContentPartImageParam(image_url=ImageURL(url='https://example.com/image.jpg'), type='image_url'),
+			],
+			cache=True,
+		)
+
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_with_image])
+
+		assert len(messages) == 1
+		assert isinstance(messages[0]['content'], list)
+		assert len(messages[0]['content']) == 2
+
+	def test_cache_with_base64_images(self):
+		"""Test cache functionality with base64 images."""
+		base64_url = 'data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='
+
+		user_with_base64 = UserMessage(
+			content=[
+				ContentPartTextParam(text='Base64 image:', type='text'),
+				ContentPartImageParam(image_url=ImageURL(url=base64_url), type='image_url'),
+			],
+			cache=True,
+		)
+
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_with_base64])
+
+		assert len(messages) == 1
+		assert isinstance(messages[0]['content'], list)
+
+	def test_cache_content_types(self):
+		"""Test different content types with cache."""
+		# String content with cache should become list
+		user_string_cached = UserMessage(content='String message', cache=True)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_string_cached])
+		assert isinstance(messages[0]['content'], list)
+
+		# String content without cache should remain string
+		user_string_no_cache = UserMessage(content='String message', cache=False)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_string_no_cache])
+		assert isinstance(messages[0]['content'], str)
+
+		# List content maintains list format regardless of cache
+		user_list_cached = UserMessage(content=[ContentPartTextParam(text='List message', type='text')], cache=True)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_list_cached])
+		assert isinstance(messages[0]['content'], list)
+
+		user_list_no_cache = UserMessage(content=[ContentPartTextParam(text='List message', type='text')], cache=False)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_list_no_cache])
+		assert isinstance(messages[0]['content'], list)
+
+	def test_assistant_cache_empty_content(self):
+		"""Test AssistantMessage with empty content and cache."""
+		# With cache
+		assistant_empty_cached = AssistantMessage(content=None, cache=True)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([assistant_empty_cached])
+
+		assert len(messages) == 1
+		assert isinstance(messages[0]['content'], list)
+
+		# Without cache
+		assistant_empty_no_cache = AssistantMessage(content=None, cache=False)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([assistant_empty_no_cache])
+
+		assert len(messages) == 1
+		assert isinstance(messages[0]['content'], str)
+
+	def test_mixed_cache_scenarios(self):
+		"""Test various combinations of cached and non-cached messages."""
+		messages_list: list[BaseMessage] = [
+			SystemMessage(content='System with cache', cache=True),
+			UserMessage(content='User with cache', cache=True),
+			AssistantMessage(content='Assistant without cache', cache=False),
+			UserMessage(content='User without cache', cache=False),
+			AssistantMessage(content='Assistant with cache', cache=True),
+		]
+
+		serialized_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages_list)
+
+		# Check system message is cached (becomes list)
+		assert isinstance(system_message, list)
+
+		# Check serialized messages
+		assert len(serialized_messages) == 4
+
+		# User with cache should be list
+		assert isinstance(serialized_messages[0]['content'], list)
+
+		# Assistant without cache should be string
+		assert isinstance(serialized_messages[1]['content'], str)
+
+		# User without cache should be string
+		assert isinstance(serialized_messages[2]['content'], str)
+
+		# Assistant with cache should be list
+		assert isinstance(serialized_messages[3]['content'], list)
+
+	def test_system_message_cache_behavior(self):
+		"""Test SystemMessage specific cache behavior."""
+		# With cache
+		system_cached = SystemMessage(content='System message with cache', cache=True)
+		result = AnthropicMessageSerializer.serialize(system_cached)
+		assert isinstance(result, SystemMessage)
+
+		# Test serialization to string format
+		serialized_content = AnthropicMessageSerializer._serialize_content_to_str(result.content, use_cache=True)
+		assert isinstance(serialized_content, list)
+
+		# Without cache
+		system_no_cache = SystemMessage(content='System message without cache', cache=False)
+		result = AnthropicMessageSerializer.serialize(system_no_cache)
+		assert isinstance(result, SystemMessage)
+
+		serialized_content = AnthropicMessageSerializer._serialize_content_to_str(result.content, use_cache=False)
+		assert isinstance(serialized_content, str)
+
+	def test_agent_messages_integration(self):
+		"""Test integration with actual agent messages."""
+		agent = Agent(task='Hello, world!', llm=ChatAnthropic(''))
+
+		messages = agent.message_manager.get_messages()
+		anthropic_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages)
+
+		# System message should be properly handled
+		assert system_message is not None
+
+	def test_cache_cleaning_last_message_only(self):
+		"""Test that only the last cache=True message remains cached."""
+		# Create multiple messages with cache=True
+		messages_list: list[BaseMessage] = [
+			UserMessage(content='First user message', cache=True),
+			AssistantMessage(content='First assistant message', cache=True),
+			UserMessage(content='Second user message', cache=True),
+			AssistantMessage(content='Second assistant message', cache=False),
+			UserMessage(content='Third user message', cache=True),  # This should be the only one cached
+		]
+
+		# Test the cleaning method directly (only accepts non-system messages)
+		normal_messages = cast(list[NonSystemMessage], [msg for msg in messages_list if not isinstance(msg, SystemMessage)])
+		cleaned_messages = AnthropicMessageSerializer._clean_cache_messages(normal_messages)
+
+		# Verify only the last cache=True message remains cached
+		assert not cleaned_messages[0].cache  # First user message should be uncached
+		assert not cleaned_messages[1].cache  # First assistant message should be uncached
+		assert not cleaned_messages[2].cache  # Second user message should be uncached
+		assert not cleaned_messages[3].cache  # Second assistant message was already uncached
+		assert cleaned_messages[4].cache  # Third user message should remain cached
+
+		# Test through serialize_messages
+		serialized_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages_list)
+
+		# Count how many messages have list content (indicating caching)
+		cached_content_count = sum(1 for msg in serialized_messages if isinstance(msg['content'], list))
+
+		# Only one message should have cached content
+		assert cached_content_count == 1
+
+		# The last message should be the cached one
+		assert isinstance(serialized_messages[-1]['content'], list)
+
+	def test_cache_cleaning_with_system_message(self):
+		"""Test that system messages are not affected by cache cleaning logic."""
+		messages_list: list[BaseMessage] = [
+			SystemMessage(content='System message', cache=True),  # System messages are handled separately
+			UserMessage(content='First user message', cache=True),
+			AssistantMessage(content='Assistant message', cache=True),  # This should be the only normal message cached
+		]
+
+		# Test through serialize_messages to see the full integration
+		serialized_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages_list)
+
+		# System message should be cached
+		assert isinstance(system_message, list)
+
+		# Only one normal message should have cached content (the last one)
+		cached_content_count = sum(1 for msg in serialized_messages if isinstance(msg['content'], list))
+		assert cached_content_count == 1
+
+		# The last message should be the cached one
+		assert isinstance(serialized_messages[-1]['content'], list)
+
+	def test_cache_cleaning_no_cached_messages(self):
+		"""Test that messages without cache=True are not affected."""
+		normal_messages_list = [
+			UserMessage(content='User message 1', cache=False),
+			AssistantMessage(content='Assistant message 1', cache=False),
+			UserMessage(content='User message 2', cache=False),
+		]
+
+		cleaned_messages = AnthropicMessageSerializer._clean_cache_messages(normal_messages_list)
+
+		# All messages should remain uncached
+		for msg in cleaned_messages:
+			assert not msg.cache
+
+	def test_max_4_cache_blocks(self):
+		"""Test that the max number of cache blocks is 4."""
+		agent = Agent(task='Hello, world!', llm=ChatAnthropic(''))
+		messages = agent.message_manager.get_messages()
+		anthropic_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages)
+
+		logger.info(anthropic_messages)
+		logger.info(system_message)
+
+
+if __name__ == '__main__':
+	test_instance = TestAnthropicCache()
+	test_instance.test_cache_basic_functionality()
+	test_instance.test_cache_with_tool_calls()
+	test_instance.test_cache_with_images()
+	test_instance.test_cache_with_base64_images()
+	test_instance.test_cache_content_types()
+	test_instance.test_assistant_cache_empty_content()
+	test_instance.test_mixed_cache_scenarios()
+	test_instance.test_system_message_cache_behavior()
+	test_instance.test_agent_messages_integration()
+	test_instance.test_cache_cleaning_last_message_only()
+	test_instance.test_cache_cleaning_with_system_message()
+	test_instance.test_cache_cleaning_no_cached_messages()
+	test_instance.test_max_4_cache_blocks()
+	print('All cache tests passed!')

chapter9/browser-use-rpa/browser-use/browser_use/llm/tests/test_chat_models.py

@@ -0,0 +1,248 @@
+import os
+
+import pytest
+from pydantic import BaseModel
+
+from browser_use.llm import ChatAnthropic, ChatGoogle, ChatGroq, ChatOpenAI, ChatOpenRouter
+from browser_use.llm.messages import ContentPartTextParam
+
+
+class CapitalResponse(BaseModel):
+	"""Structured response for capital question"""
+
+	country: str
+	capital: str
+
+
+class TestChatModels:
+	from browser_use.llm.messages import (
+		AssistantMessage,
+		BaseMessage,
+		SystemMessage,
+		UserMessage,
+	)
+
+	"""Test suite for all chat model implementations"""
+
+	# Test Constants
+	SYSTEM_MESSAGE = SystemMessage(content=[ContentPartTextParam(text='You are a helpful assistant.', type='text')])
+	FRANCE_QUESTION = UserMessage(content='What is the capital of France? Answer in one word.')
+	FRANCE_ANSWER = AssistantMessage(content='Paris')
+	GERMANY_QUESTION = UserMessage(content='What is the capital of Germany? Answer in one word.')
+
+	# Expected values
+	EXPECTED_GERMANY_CAPITAL = 'berlin'
+	EXPECTED_FRANCE_COUNTRY = 'france'
+	EXPECTED_FRANCE_CAPITAL = 'paris'
+
+	# Test messages for conversation
+	CONVERSATION_MESSAGES: list[BaseMessage] = [
+		SYSTEM_MESSAGE,
+		FRANCE_QUESTION,
+		FRANCE_ANSWER,
+		GERMANY_QUESTION,
+	]
+
+	# Test messages for structured output
+	STRUCTURED_MESSAGES: list[BaseMessage] = [UserMessage(content='What is the capital of France?')]
+
+	# OpenAI Tests
+	@pytest.fixture
+	def openrouter_chat(self):
+		"""Provides an initialized ChatOpenRouter client for tests."""
+		if not os.getenv('OPENROUTER_API_KEY'):
+			pytest.skip('OPENROUTER_API_KEY not set')
+		return ChatOpenRouter(model='openai/gpt-4o-mini', api_key=os.getenv('OPENROUTER_API_KEY'), temperature=0)
+
+	@pytest.mark.asyncio
+	async def test_openai_ainvoke_normal(self):
+		"""Test normal text response from OpenAI"""
+		# Skip if no API key
+		if not os.getenv('OPENAI_API_KEY'):
+			pytest.skip('OPENAI_API_KEY not set')
+
+		chat = ChatOpenAI(model='gpt-4o-mini', temperature=0)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_openai_ainvoke_structured(self):
+		"""Test structured output from OpenAI"""
+		# Skip if no API key
+		if not os.getenv('OPENAI_API_KEY'):
+			pytest.skip('OPENAI_API_KEY not set')
+
+		chat = ChatOpenAI(model='gpt-4o-mini', temperature=0)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL
+
+	# Anthropic Tests
+	@pytest.mark.asyncio
+	async def test_anthropic_ainvoke_normal(self):
+		"""Test normal text response from Anthropic"""
+		# Skip if no API key
+		if not os.getenv('ANTHROPIC_API_KEY'):
+			pytest.skip('ANTHROPIC_API_KEY not set')
+
+		chat = ChatAnthropic(model='claude-3-5-haiku-latest', max_tokens=100, temperature=0)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_anthropic_ainvoke_structured(self):
+		"""Test structured output from Anthropic"""
+		# Skip if no API key
+		if not os.getenv('ANTHROPIC_API_KEY'):
+			pytest.skip('ANTHROPIC_API_KEY not set')
+
+		chat = ChatAnthropic(model='claude-3-5-haiku-latest', max_tokens=100, temperature=0)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL
+
+	# Google Gemini Tests
+	@pytest.mark.asyncio
+	async def test_google_ainvoke_normal(self):
+		"""Test normal text response from Google Gemini"""
+		# Skip if no API key
+		if not os.getenv('GOOGLE_API_KEY'):
+			pytest.skip('GOOGLE_API_KEY not set')
+
+		chat = ChatGoogle(model='gemini-2.0-flash', api_key=os.getenv('GOOGLE_API_KEY'), temperature=0)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_google_ainvoke_structured(self):
+		"""Test structured output from Google Gemini"""
+		# Skip if no API key
+		if not os.getenv('GOOGLE_API_KEY'):
+			pytest.skip('GOOGLE_API_KEY not set')
+
+		chat = ChatGoogle(model='gemini-2.0-flash', api_key=os.getenv('GOOGLE_API_KEY'), temperature=0)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL
+
+	# Google Gemini with Vertex AI Tests
+	@pytest.mark.asyncio
+	async def test_google_vertex_ainvoke_normal(self):
+		"""Test normal text response from Google Gemini via Vertex AI"""
+		# Skip if no project ID
+		if not os.getenv('GOOGLE_CLOUD_PROJECT'):
+			pytest.skip('GOOGLE_CLOUD_PROJECT not set')
+
+		chat = ChatGoogle(
+			model='gemini-2.0-flash',
+			vertexai=True,
+			project=os.getenv('GOOGLE_CLOUD_PROJECT'),
+			location='us-central1',
+			temperature=0,
+		)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_google_vertex_ainvoke_structured(self):
+		"""Test structured output from Google Gemini via Vertex AI"""
+		# Skip if no project ID
+		if not os.getenv('GOOGLE_CLOUD_PROJECT'):
+			pytest.skip('GOOGLE_CLOUD_PROJECT not set')
+
+		chat = ChatGoogle(
+			model='gemini-2.0-flash',
+			vertexai=True,
+			project=os.getenv('GOOGLE_CLOUD_PROJECT'),
+			location='us-central1',
+			temperature=0,
+		)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL
+
+	# Groq Tests
+	@pytest.mark.asyncio
+	async def test_groq_ainvoke_normal(self):
+		"""Test normal text response from Groq"""
+		# Skip if no API key
+		if not os.getenv('GROQ_API_KEY'):
+			pytest.skip('GROQ_API_KEY not set')
+
+		chat = ChatGroq(model='meta-llama/llama-4-maverick-17b-128e-instruct', temperature=0)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_groq_ainvoke_structured(self):
+		"""Test structured output from Groq"""
+		# Skip if no API key
+		if not os.getenv('GROQ_API_KEY'):
+			pytest.skip('GROQ_API_KEY not set')
+
+		chat = ChatGroq(model='meta-llama/llama-4-maverick-17b-128e-instruct', temperature=0)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL
+
+	# OpenRouter Tests
+	@pytest.mark.asyncio
+	async def test_openrouter_ainvoke_normal(self):
+		"""Test normal text response from OpenRouter"""
+		# Skip if no API key
+		if not os.getenv('OPENROUTER_API_KEY'):
+			pytest.skip('OPENROUTER_API_KEY not set')
+
+		chat = ChatOpenRouter(model='openai/gpt-4o-mini', api_key=os.getenv('OPENROUTER_API_KEY'), temperature=0)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_openrouter_ainvoke_structured(self):
+		"""Test structured output from OpenRouter"""
+		# Skip if no API key
+		if not os.getenv('OPENROUTER_API_KEY'):
+			pytest.skip('OPENROUTER_API_KEY not set')
+
+		chat = ChatOpenRouter(model='openai/gpt-4o-mini', api_key=os.getenv('OPENROUTER_API_KEY'), temperature=0)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL

chapter9/browser-use-rpa/browser-use/browser_use/llm/tests/test_gemini_image.py

@@ -0,0 +1,91 @@
+import asyncio
+import base64
+import io
+import random
+
+from PIL import Image, ImageDraw, ImageFont
+
+from browser_use.llm.google.chat import ChatGoogle
+from browser_use.llm.google.serializer import GoogleMessageSerializer
+from browser_use.llm.messages import (
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	ImageURL,
+	SystemMessage,
+	UserMessage,
+)
+
+
+def create_random_text_image(text: str = 'hello world', width: int = 4000, height: int = 4000) -> str:
+	# Create image with random background color
+	bg_color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
+	image = Image.new('RGB', (width, height), bg_color)
+	draw = ImageDraw.Draw(image)
+
+	# Try to use a default font, fallback to default if not available
+	try:
+		font = ImageFont.truetype('arial.ttf', 24)
+	except Exception:
+		font = ImageFont.load_default()
+
+	# Calculate text position to center it
+	bbox = draw.textbbox((0, 0), text, font=font)
+	text_width = bbox[2] - bbox[0]
+	text_height = bbox[3] - bbox[1]
+	x = (width - text_width) // 2
+	y = (height - text_height) // 2
+
+	# Draw text with contrasting color
+	text_color = (255 - bg_color[0], 255 - bg_color[1], 255 - bg_color[2])
+	draw.text((x, y), text, fill=text_color, font=font)
+
+	# Convert to base64
+	buffer = io.BytesIO()
+	image.save(buffer, format='PNG')
+	img_data = base64.b64encode(buffer.getvalue()).decode()
+
+	return f'data:image/png;base64,{img_data}'
+
+
+async def test_gemini_image_vision():
+	"""Test Gemini's ability to see and describe images."""
+
+	# Create the LLM
+	llm = ChatGoogle(model='gemini-2.0-flash-exp')
+
+	# Create a random image with text
+	image_data_url = create_random_text_image('Hello Gemini! Can you see this text?')
+
+	# Create messages with image
+	messages: list[BaseMessage] = [
+		SystemMessage(content='You are a helpful assistant that can see and describe images.'),
+		UserMessage(
+			content=[
+				ContentPartTextParam(text='What do you see in this image? Please describe the text and any visual elements.'),
+				ContentPartImageParam(image_url=ImageURL(url=image_data_url)),
+			]
+		),
+	]
+
+	# Serialize messages for Google format
+	serializer = GoogleMessageSerializer()
+	formatted_messages, system_message = serializer.serialize_messages(messages)
+
+	print('Testing Gemini image vision...')
+	print(f'System message: {system_message}')
+
+	# Make the API call
+	try:
+		response = await llm.ainvoke(messages)
+		print('\n=== Gemini Response ===')
+		print(response.completion)
+		print(response.usage)
+		print('=======================')
+	except Exception as e:
+		print(f'Error calling Gemini: {e}')
+		print(f'Error type: {type(e)}')
+
+
+if __name__ == '__main__':
+	asyncio.run(test_gemini_image_vision())

chapter9/browser-use-rpa/browser-use/browser_use/llm/tests/test_groq_loop.py

@@ -0,0 +1,51 @@
+import asyncio
+
+from browser_use.llm import ContentText
+from browser_use.llm.groq.chat import ChatGroq
+from browser_use.llm.messages import SystemMessage, UserMessage
+
+llm = ChatGroq(
+	model='meta-llama/llama-4-maverick-17b-128e-instruct',
+	temperature=0.5,
+)
+# llm = ChatOpenAI(model='gpt-4.1-mini')
+
+
+async def main():
+	from pydantic import BaseModel
+
+	from browser_use.tokens.service import TokenCost
+
+	tk = TokenCost().register_llm(llm)
+
+	class Output(BaseModel):
+		reasoning: str
+		answer: str
+
+	message = [
+		SystemMessage(content='You are a helpful assistant that can answer questions and help with tasks.'),
+		UserMessage(
+			content=[
+				ContentText(
+					text=r"Why is the sky blue? write exactly this into reasoning make sure to output ' with  exactly like in the input : "
+				),
+				ContentText(
+					text="""
+	The user's request is to find the lowest priced women's plus size one piece swimsuit in color black with a customer rating of at least 5 on Kohls.com. I am currently on the homepage of Kohls. The page has a search bar and various category links. To begin, I need to navigate to the women's section and search for swimsuits. I will start by clicking on the 'Women' category link."""
+				),
+			]
+		),
+	]
+
+	for i in range(10):
+		print('-' * 50)
+		print(f'start loop {i}')
+		response = await llm.ainvoke(message, output_format=Output)
+		completion = response.completion
+		print(f'start reasoning: {completion.reasoning}')
+		print(f'answer: {completion.answer}')
+		print('-' * 50)
+
+
+if __name__ == '__main__':
+	asyncio.run(main())

chapter9/browser-use-rpa/browser-use/browser_use/llm/tests/test_single_step.py

@@ -0,0 +1,188 @@
+import logging
+import tempfile
+
+import pytest
+
+from browser_use.agent.prompts import AgentMessagePrompt
+from browser_use.agent.service import Agent
+from browser_use.browser.views import BrowserStateSummary, TabInfo
+from browser_use.dom.views import DOMSelectorMap, EnhancedDOMTreeNode, NodeType, SerializedDOMState, SimplifiedNode
+from browser_use.filesystem.file_system import FileSystem
+from browser_use.llm.anthropic.chat import ChatAnthropic
+from browser_use.llm.azure.chat import ChatAzureOpenAI
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.google.chat import ChatGoogle
+from browser_use.llm.groq.chat import ChatGroq
+from browser_use.llm.openai.chat import ChatOpenAI
+
+# Set logging level to INFO for this module
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def create_mock_state_message(temp_dir: str):
+	"""Create a mock state message with a single clickable element."""
+
+	# Create a mock DOM element with a single clickable button
+	mock_button = EnhancedDOMTreeNode(
+		node_id=1,
+		backend_node_id=1,
+		node_type=NodeType.ELEMENT_NODE,
+		node_name='button',
+		node_value='Click Me',
+		attributes={'id': 'test-button'},
+		is_scrollable=False,
+		is_visible=True,
+		absolute_position=None,
+		session_id=None,
+		target_id='ABCD1234ABCD1234ABCD1234ABCD1234ABCD1234',
+		frame_id=None,
+		content_document=None,
+		shadow_root_type=None,
+		shadow_roots=None,
+		parent_node=None,
+		children_nodes=None,
+		ax_node=None,
+		snapshot_node=None,
+	)
+
+	# Create selector map
+	selector_map: DOMSelectorMap = {1: mock_button}
+
+	# Create mock tab info with proper target_id
+	mock_tab = TabInfo(
+		target_id='ABCD1234ABCD1234ABCD1234ABCD1234ABCD1234',
+		url='https://example.com',
+		title='Test Page',
+	)
+
+	dom_state = SerializedDOMState(
+		_root=SimplifiedNode(
+			original_node=mock_button,
+			children=[],
+			should_display=True,
+			interactive_index=1,
+		),
+		selector_map=selector_map,
+	)
+
+	# Create mock browser state with required selector_map
+	mock_browser_state = BrowserStateSummary(
+		dom_state=dom_state,  # Using the actual DOM element
+		url='https://example.com',
+		title='Test Page',
+		tabs=[mock_tab],
+		screenshot='',  # Empty screenshot
+		pixels_above=0,
+		pixels_below=0,
+	)
+
+	# Create file system using the provided temp directory
+	mock_file_system = FileSystem(temp_dir)
+
+	# Create the agent message prompt
+	agent_prompt = AgentMessagePrompt(
+		browser_state_summary=mock_browser_state,
+		file_system=mock_file_system,  # Now using actual FileSystem instance
+		agent_history_description='',  # Empty history
+		read_state_description='',  # Empty read state
+		task='Click the button on the page',
+		include_attributes=['id'],
+		step_info=None,
+		page_filtered_actions=None,
+		max_clickable_elements_length=40000,
+		sensitive_data=None,
+	)
+
+	# Override the clickable_elements_to_string method to return our simple element
+	dom_state.llm_representation = lambda include_attributes=None: '[1]<button id="test-button">Click Me</button>'
+
+	# Get the formatted message
+	message = agent_prompt.get_user_message(use_vision=False)
+
+	return message
+
+
+# Pytest parameterized version
+@pytest.mark.parametrize(
+	'llm_class,model_name',
+	[
+		(ChatGroq, 'meta-llama/llama-4-maverick-17b-128e-instruct'),
+		(ChatGoogle, 'gemini-2.0-flash-exp'),
+		(ChatOpenAI, 'gpt-4.1-mini'),
+		(ChatAnthropic, 'claude-3-5-sonnet-latest'),
+		(ChatAzureOpenAI, 'gpt-4.1-mini'),
+	],
+)
+async def test_single_step_parametrized(llm_class, model_name):
+	"""Test single step with different LLM providers using pytest parametrize."""
+	llm = llm_class(model=model_name)
+
+	agent = Agent(task='Click the button on the page', llm=llm)
+
+	# Create temporary directory that will stay alive during the test
+	with tempfile.TemporaryDirectory() as temp_dir:
+		# Create mock state message
+		mock_message = create_mock_state_message(temp_dir)
+
+		agent.message_manager._set_message_with_type(mock_message, 'state')
+
+		messages = agent.message_manager.get_messages()
+
+		# Test with simple question
+		response = await llm.ainvoke(messages, agent.AgentOutput)
+
+		# Basic assertions to ensure response is valid
+		assert response.completion is not None
+		assert response.usage is not None
+		assert response.usage.total_tokens > 0
+
+
+async def test_single_step():
+	"""Original test function that tests all models in a loop."""
+	# Create a list of models to test
+	models: list[BaseChatModel] = [
+		ChatGroq(model='meta-llama/llama-4-maverick-17b-128e-instruct'),
+		ChatGoogle(model='gemini-2.0-flash-exp'),
+		ChatOpenAI(model='gpt-4.1'),
+		ChatAnthropic(model='claude-3-5-sonnet-latest'),  # Using haiku for cost efficiency
+		ChatAzureOpenAI(model='gpt-4o-mini'),
+	]
+
+	for llm in models:
+		print(f'\n{"=" * 60}')
+		print(f'Testing with model: {llm.provider} - {llm.model}')
+		print(f'{"=" * 60}\n')
+
+		agent = Agent(task='Click the button on the page', llm=llm)
+
+		# Create temporary directory that will stay alive during the test
+		with tempfile.TemporaryDirectory() as temp_dir:
+			# Create mock state message
+			mock_message = create_mock_state_message(temp_dir)
+
+			# Print the mock message content to see what it looks like
+			print('Mock state message:')
+			print(mock_message.content)
+			print('\n' + '=' * 50 + '\n')
+
+			agent.message_manager._set_message_with_type(mock_message, 'state')
+
+			messages = agent.message_manager.get_messages()
+
+			# Test with simple question
+			try:
+				response = await llm.ainvoke(messages, agent.AgentOutput)
+				logger.info(f'Response from {llm.provider}: {response.completion}')
+				logger.info(f'Actions: {str(response.completion.action)}')
+
+			except Exception as e:
+				logger.error(f'Error with {llm.provider}: {type(e).__name__}: {str(e)}')
+
+		print(f'\n{"=" * 60}\n')
+
+
+if __name__ == '__main__':
+	import asyncio
+
+	asyncio.run(test_single_step())