ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter9/browser-use-rpa/browser-use/browser_use/llm/README.md

@@ -0,0 +1,16 @@
+# Browser Use LLMs
+
+We officially support the following LLMs:
+
+- OpenAI
+- Anthropic
+- Google
+- Groq
+- Ollama
+- DeepSeek
+
+## Migrating from LangChain
+
+Because of how we implemented the LLMs, we can technically support anything. If you want to use a LangChain model, you can use the `ChatLangchain` (NOT OFFICIALLY SUPPORTED) class.
+
+You can find all the details in the [LangChain example](examples/models/langchain/example.py). We suggest you grab that code and use it as a reference.

chapter9/browser-use-rpa/browser-use/browser_use/llm/__init__.py

@@ -0,0 +1,146 @@
+"""
+We have switched all of our code from langchain to openai.types.chat.chat_completion_message_param.
+
+For easier transition we have
+"""
+
+from typing import TYPE_CHECKING
+
+# Lightweight imports that are commonly used
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	SystemMessage,
+	UserMessage,
+)
+from browser_use.llm.messages import (
+	ContentPartImageParam as ContentImage,
+)
+from browser_use.llm.messages import (
+	ContentPartRefusalParam as ContentRefusal,
+)
+from browser_use.llm.messages import (
+	ContentPartTextParam as ContentText,
+)
+
+# Type stubs for lazy imports
+if TYPE_CHECKING:
+	from browser_use.llm.anthropic.chat import ChatAnthropic
+	from browser_use.llm.aws.chat_anthropic import ChatAnthropicBedrock
+	from browser_use.llm.aws.chat_bedrock import ChatAWSBedrock
+	from browser_use.llm.azure.chat import ChatAzureOpenAI
+	from browser_use.llm.deepseek.chat import ChatDeepSeek
+	from browser_use.llm.google.chat import ChatGoogle
+	from browser_use.llm.groq.chat import ChatGroq
+	from browser_use.llm.ollama.chat import ChatOllama
+	from browser_use.llm.openai.chat import ChatOpenAI
+	from browser_use.llm.openrouter.chat import ChatOpenRouter
+
+	# Type stubs for model instances - enables IDE autocomplete
+	openai_gpt_4o: ChatOpenAI
+	openai_gpt_4o_mini: ChatOpenAI
+	openai_gpt_4_1_mini: ChatOpenAI
+	openai_o1: ChatOpenAI
+	openai_o1_mini: ChatOpenAI
+	openai_o1_pro: ChatOpenAI
+	openai_o3: ChatOpenAI
+	openai_o3_mini: ChatOpenAI
+	openai_o3_pro: ChatOpenAI
+	openai_o4_mini: ChatOpenAI
+	openai_gpt_5: ChatOpenAI
+	openai_gpt_5_mini: ChatOpenAI
+	openai_gpt_5_nano: ChatOpenAI
+
+	azure_gpt_4o: ChatAzureOpenAI
+	azure_gpt_4o_mini: ChatAzureOpenAI
+	azure_gpt_4_1_mini: ChatAzureOpenAI
+	azure_o1: ChatAzureOpenAI
+	azure_o1_mini: ChatAzureOpenAI
+	azure_o1_pro: ChatAzureOpenAI
+	azure_o3: ChatAzureOpenAI
+	azure_o3_mini: ChatAzureOpenAI
+	azure_o3_pro: ChatAzureOpenAI
+	azure_gpt_5: ChatAzureOpenAI
+	azure_gpt_5_mini: ChatAzureOpenAI
+
+	google_gemini_2_0_flash: ChatGoogle
+	google_gemini_2_0_pro: ChatGoogle
+	google_gemini_2_5_pro: ChatGoogle
+	google_gemini_2_5_flash: ChatGoogle
+	google_gemini_2_5_flash_lite: ChatGoogle
+
+# Models are imported on-demand via __getattr__
+
+# Lazy imports mapping for heavy chat models
+_LAZY_IMPORTS = {
+	'ChatAnthropic': ('browser_use.llm.anthropic.chat', 'ChatAnthropic'),
+	'ChatAnthropicBedrock': ('browser_use.llm.aws.chat_anthropic', 'ChatAnthropicBedrock'),
+	'ChatAWSBedrock': ('browser_use.llm.aws.chat_bedrock', 'ChatAWSBedrock'),
+	'ChatAzureOpenAI': ('browser_use.llm.azure.chat', 'ChatAzureOpenAI'),
+	'ChatDeepSeek': ('browser_use.llm.deepseek.chat', 'ChatDeepSeek'),
+	'ChatGoogle': ('browser_use.llm.google.chat', 'ChatGoogle'),
+	'ChatGroq': ('browser_use.llm.groq.chat', 'ChatGroq'),
+	'ChatOllama': ('browser_use.llm.ollama.chat', 'ChatOllama'),
+	'ChatOpenAI': ('browser_use.llm.openai.chat', 'ChatOpenAI'),
+	'ChatOpenRouter': ('browser_use.llm.openrouter.chat', 'ChatOpenRouter'),
+}
+
+# Cache for model instances - only created when accessed
+_model_cache: dict[str, 'BaseChatModel'] = {}
+
+
+def __getattr__(name: str):
+	"""Lazy import mechanism for heavy chat model imports and model instances."""
+	if name in _LAZY_IMPORTS:
+		module_path, attr_name = _LAZY_IMPORTS[name]
+		try:
+			from importlib import import_module
+
+			module = import_module(module_path)
+			attr = getattr(module, attr_name)
+			return attr
+		except ImportError as e:
+			raise ImportError(f'Failed to import {name} from {module_path}: {e}') from e
+
+	# Check cache first for model instances
+	if name in _model_cache:
+		return _model_cache[name]
+
+	# Try to get model instances from models module on-demand
+	try:
+		from browser_use.llm.models import __getattr__ as models_getattr
+
+		attr = models_getattr(name)
+		# Cache in our clean cache dict
+		_model_cache[name] = attr
+		return attr
+	except (AttributeError, ImportError):
+		pass
+
+	raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+	# Message types -> for easier transition from langchain
+	'BaseMessage',
+	'UserMessage',
+	'SystemMessage',
+	'AssistantMessage',
+	# Content parts with better names
+	'ContentText',
+	'ContentRefusal',
+	'ContentImage',
+	# Chat models
+	'BaseChatModel',
+	'ChatOpenAI',
+	'ChatDeepSeek',
+	'ChatGoogle',
+	'ChatAnthropic',
+	'ChatAnthropicBedrock',
+	'ChatAWSBedrock',
+	'ChatGroq',
+	'ChatAzureOpenAI',
+	'ChatOllama',
+	'ChatOpenRouter',
+]

chapter9/browser-use-rpa/browser-use/browser_use/llm/anthropic/chat.py

@@ -0,0 +1,236 @@
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any, TypeVar, overload
+
+import httpx
+from anthropic import (
+	NOT_GIVEN,
+	APIConnectionError,
+	APIStatusError,
+	AsyncAnthropic,
+	NotGiven,
+	RateLimitError,
+)
+from anthropic.types import CacheControlEphemeralParam, Message, ToolParam
+from anthropic.types.model_param import ModelParam
+from anthropic.types.text_block import TextBlock
+from anthropic.types.tool_choice_tool_param import ToolChoiceToolParam
+from httpx import Timeout
+from pydantic import BaseModel
+
+from browser_use.llm.anthropic.serializer import AnthropicMessageSerializer
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatAnthropic(BaseChatModel):
+	"""
+	A wrapper around Anthropic's chat model.
+	"""
+
+	# Model configuration
+	model: str | ModelParam
+	max_tokens: int = 8192
+	temperature: float | None = None
+	top_p: float | None = None
+	seed: int | None = None
+
+	# Client initialization parameters
+	api_key: str | None = None
+	auth_token: str | None = None
+	base_url: str | httpx.URL | None = None
+	timeout: float | Timeout | None | NotGiven = NotGiven()
+	max_retries: int = 10
+	default_headers: Mapping[str, str] | None = None
+	default_query: Mapping[str, object] | None = None
+
+	# Static
+	@property
+	def provider(self) -> str:
+		return 'anthropic'
+
+	def _get_client_params(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary."""
+		# Define base client params
+		base_params = {
+			'api_key': self.api_key,
+			'auth_token': self.auth_token,
+			'base_url': self.base_url,
+			'timeout': self.timeout,
+			'max_retries': self.max_retries,
+			'default_headers': self.default_headers,
+			'default_query': self.default_query,
+		}
+
+		# Create client_params dict with non-None values and non-NotGiven values
+		client_params = {}
+		for k, v in base_params.items():
+			if v is not None and v is not NotGiven():
+				client_params[k] = v
+
+		return client_params
+
+	def _get_client_params_for_invoke(self):
+		"""Prepare client parameters dictionary for invoke."""
+
+		client_params = {}
+
+		if self.temperature is not None:
+			client_params['temperature'] = self.temperature
+
+		if self.max_tokens is not None:
+			client_params['max_tokens'] = self.max_tokens
+
+		if self.top_p is not None:
+			client_params['top_p'] = self.top_p
+
+		if self.seed is not None:
+			client_params['seed'] = self.seed
+
+		return client_params
+
+	def get_client(self) -> AsyncAnthropic:
+		"""
+		Returns an AsyncAnthropic client.
+
+		Returns:
+			AsyncAnthropic: An instance of the AsyncAnthropic client.
+		"""
+		client_params = self._get_client_params()
+		return AsyncAnthropic(**client_params)
+
+	@property
+	def name(self) -> str:
+		return str(self.model)
+
+	def _get_usage(self, response: Message) -> ChatInvokeUsage | None:
+		usage = ChatInvokeUsage(
+			prompt_tokens=response.usage.input_tokens
+			+ (
+				response.usage.cache_read_input_tokens or 0
+			),  # Total tokens in Anthropic are a bit fucked, you have to add cached tokens to the prompt tokens
+			completion_tokens=response.usage.output_tokens,
+			total_tokens=response.usage.input_tokens + response.usage.output_tokens,
+			prompt_cached_tokens=response.usage.cache_read_input_tokens,
+			prompt_cache_creation_tokens=response.usage.cache_creation_input_tokens,
+			prompt_image_tokens=None,
+		)
+		return usage
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: None = None) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T]) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		anthropic_messages, system_prompt = AnthropicMessageSerializer.serialize_messages(messages)
+
+		try:
+			if output_format is None:
+				# Normal completion without structured output
+				response = await self.get_client().messages.create(
+					model=self.model,
+					messages=anthropic_messages,
+					system=system_prompt or NOT_GIVEN,
+					**self._get_client_params_for_invoke(),
+				)
+
+				# Ensure we have a valid Message object before accessing attributes
+				if not isinstance(response, Message):
+					raise ModelProviderError(
+						message=f'Unexpected response type from Anthropic API: {type(response).__name__}. Response: {str(response)[:200]}',
+						status_code=502,
+						model=self.name,
+					)
+
+				usage = self._get_usage(response)
+
+				# Extract text from the first content block
+				first_content = response.content[0]
+				if isinstance(first_content, TextBlock):
+					response_text = first_content.text
+				else:
+					# If it's not a text block, convert to string
+					response_text = str(first_content)
+
+				return ChatInvokeCompletion(
+					completion=response_text,
+					usage=usage,
+				)
+
+			else:
+				# Use tool calling for structured output
+				# Create a tool that represents the output format
+				tool_name = output_format.__name__
+				schema = SchemaOptimizer.create_optimized_json_schema(output_format)
+
+				# Remove title from schema if present (Anthropic doesn't like it in parameters)
+				if 'title' in schema:
+					del schema['title']
+
+				tool = ToolParam(
+					name=tool_name,
+					description=f'Extract information in the format of {tool_name}',
+					input_schema=schema,
+					cache_control=CacheControlEphemeralParam(type='ephemeral'),
+				)
+
+				# Force the model to use this tool
+				tool_choice = ToolChoiceToolParam(type='tool', name=tool_name)
+
+				response = await self.get_client().messages.create(
+					model=self.model,
+					messages=anthropic_messages,
+					tools=[tool],
+					system=system_prompt or NOT_GIVEN,
+					tool_choice=tool_choice,
+					**self._get_client_params_for_invoke(),
+				)
+
+				# Ensure we have a valid Message object before accessing attributes
+				if not isinstance(response, Message):
+					raise ModelProviderError(
+						message=f'Unexpected response type from Anthropic API: {type(response).__name__}. Response: {str(response)[:200]}',
+						status_code=502,
+						model=self.name,
+					)
+
+				usage = self._get_usage(response)
+
+				# Extract the tool use block
+				for content_block in response.content:
+					if hasattr(content_block, 'type') and content_block.type == 'tool_use':
+						# Parse the tool input as the structured output
+						try:
+							return ChatInvokeCompletion(completion=output_format.model_validate(content_block.input), usage=usage)
+						except Exception as e:
+							# If validation fails, try to parse it as JSON first
+							if isinstance(content_block.input, str):
+								data = json.loads(content_block.input)
+								return ChatInvokeCompletion(
+									completion=output_format.model_validate(data),
+									usage=usage,
+								)
+							raise e
+
+				# If no tool use block found, raise an error
+				raise ValueError('Expected tool use in response but none found')
+
+		except APIConnectionError as e:
+			raise ModelProviderError(message=e.message, model=self.name) from e
+		except RateLimitError as e:
+			raise ModelRateLimitError(message=e.message, model=self.name) from e
+		except APIStatusError as e:
+			raise ModelProviderError(message=e.message, status_code=e.status_code, model=self.name) from e
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e

chapter9/browser-use-rpa/browser-use/browser_use/llm/anthropic/serializer.py

@@ -0,0 +1,312 @@
+import json
+from typing import overload
+
+from anthropic.types import (
+	Base64ImageSourceParam,
+	CacheControlEphemeralParam,
+	ImageBlockParam,
+	MessageParam,
+	TextBlockParam,
+	ToolUseBlockParam,
+	URLImageSourceParam,
+)
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	SupportedImageMediaType,
+	SystemMessage,
+	UserMessage,
+)
+
+NonSystemMessage = UserMessage | AssistantMessage
+
+
+class AnthropicMessageSerializer:
+	"""Serializer for converting between custom message types and Anthropic message param types."""
+
+	@staticmethod
+	def _is_base64_image(url: str) -> bool:
+		"""Check if the URL is a base64 encoded image."""
+		return url.startswith('data:image/')
+
+	@staticmethod
+	def _parse_base64_url(url: str) -> tuple[SupportedImageMediaType, str]:
+		"""Parse a base64 data URL to extract media type and data."""
+		# Format: data:image/jpeg;base64,<data>
+		if not url.startswith('data:'):
+			raise ValueError(f'Invalid base64 URL: {url}')
+
+		header, data = url.split(',', 1)
+		media_type = header.split(';')[0].replace('data:', '')
+
+		# Ensure it's a supported media type
+		supported_types = ['image/jpeg', 'image/png', 'image/gif', 'image/webp']
+		if media_type not in supported_types:
+			# Default to png if not recognized
+			media_type = 'image/png'
+
+		return media_type, data  # type: ignore
+
+	@staticmethod
+	def _serialize_cache_control(use_cache: bool) -> CacheControlEphemeralParam | None:
+		"""Serialize cache control."""
+		if use_cache:
+			return CacheControlEphemeralParam(type='ephemeral')
+		return None
+
+	@staticmethod
+	def _serialize_content_part_text(part: ContentPartTextParam, use_cache: bool) -> TextBlockParam:
+		"""Convert a text content part to Anthropic's TextBlockParam."""
+		return TextBlockParam(
+			text=part.text, type='text', cache_control=AnthropicMessageSerializer._serialize_cache_control(use_cache)
+		)
+
+	@staticmethod
+	def _serialize_content_part_image(part: ContentPartImageParam) -> ImageBlockParam:
+		"""Convert an image content part to Anthropic's ImageBlockParam."""
+		url = part.image_url.url
+
+		if AnthropicMessageSerializer._is_base64_image(url):
+			# Handle base64 encoded images
+			media_type, data = AnthropicMessageSerializer._parse_base64_url(url)
+			return ImageBlockParam(
+				source=Base64ImageSourceParam(
+					data=data,
+					media_type=media_type,
+					type='base64',
+				),
+				type='image',
+			)
+		else:
+			# Handle URL images
+			return ImageBlockParam(source=URLImageSourceParam(url=url, type='url'), type='image')
+
+	@staticmethod
+	def _serialize_content_to_str(
+		content: str | list[ContentPartTextParam], use_cache: bool = False
+	) -> list[TextBlockParam] | str:
+		"""Serialize content to a string."""
+		cache_control = AnthropicMessageSerializer._serialize_cache_control(use_cache)
+
+		if isinstance(content, str):
+			if cache_control:
+				return [TextBlockParam(text=content, type='text', cache_control=cache_control)]
+			else:
+				return content
+
+		serialized_blocks: list[TextBlockParam] = []
+		for part in content:
+			if part.type == 'text':
+				serialized_blocks.append(AnthropicMessageSerializer._serialize_content_part_text(part, use_cache))
+
+		return serialized_blocks
+
+	@staticmethod
+	def _serialize_content(
+		content: str | list[ContentPartTextParam | ContentPartImageParam],
+		use_cache: bool = False,
+	) -> str | list[TextBlockParam | ImageBlockParam]:
+		"""Serialize content to Anthropic format."""
+		if isinstance(content, str):
+			if use_cache:
+				return [TextBlockParam(text=content, type='text', cache_control=CacheControlEphemeralParam(type='ephemeral'))]
+			else:
+				return content
+
+		serialized_blocks: list[TextBlockParam | ImageBlockParam] = []
+		for part in content:
+			if part.type == 'text':
+				serialized_blocks.append(AnthropicMessageSerializer._serialize_content_part_text(part, use_cache))
+			elif part.type == 'image_url':
+				serialized_blocks.append(AnthropicMessageSerializer._serialize_content_part_image(part))
+
+		return serialized_blocks
+
+	@staticmethod
+	def _serialize_tool_calls_to_content(tool_calls, use_cache: bool = False) -> list[ToolUseBlockParam]:
+		"""Convert tool calls to Anthropic's ToolUseBlockParam format."""
+		blocks: list[ToolUseBlockParam] = []
+		for tool_call in tool_calls:
+			# Parse the arguments JSON string to object
+
+			try:
+				input_obj = json.loads(tool_call.function.arguments)
+			except json.JSONDecodeError:
+				# If arguments aren't valid JSON, use as string
+				input_obj = {'arguments': tool_call.function.arguments}
+
+			blocks.append(
+				ToolUseBlockParam(
+					id=tool_call.id,
+					input=input_obj,
+					name=tool_call.function.name,
+					type='tool_use',
+					cache_control=AnthropicMessageSerializer._serialize_cache_control(use_cache),
+				)
+			)
+		return blocks
+
+	# region - Serialize overloads
+	@overload
+	@staticmethod
+	def serialize(message: UserMessage) -> MessageParam: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: SystemMessage) -> SystemMessage: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: AssistantMessage) -> MessageParam: ...
+
+	@staticmethod
+	def serialize(message: BaseMessage) -> MessageParam | SystemMessage:
+		"""Serialize a custom message to an Anthropic MessageParam.
+
+		Note: Anthropic doesn't have a 'system' role. System messages should be
+		handled separately as the system parameter in the API call, not as a message.
+		If a SystemMessage is passed here, it will be converted to a user message.
+		"""
+		if isinstance(message, UserMessage):
+			content = AnthropicMessageSerializer._serialize_content(message.content, use_cache=message.cache)
+			return MessageParam(role='user', content=content)
+
+		elif isinstance(message, SystemMessage):
+			# Anthropic doesn't have system messages in the messages array
+			# System prompts are passed separately. Convert to user message.
+			return message
+
+		elif isinstance(message, AssistantMessage):
+			# Handle content and tool calls
+			blocks: list[TextBlockParam | ToolUseBlockParam] = []
+
+			# Add content blocks if present
+			if message.content is not None:
+				if isinstance(message.content, str):
+					blocks.append(
+						TextBlockParam(
+							text=message.content,
+							type='text',
+							cache_control=AnthropicMessageSerializer._serialize_cache_control(message.cache),
+						)
+					)
+				else:
+					# Process content parts (text and refusal)
+					for part in message.content:
+						if part.type == 'text':
+							blocks.append(AnthropicMessageSerializer._serialize_content_part_text(part, use_cache=message.cache))
+						# # Note: Anthropic doesn't have a specific refusal block type,
+						# # so we convert refusals to text blocks
+						# elif part.type == 'refusal':
+						# 	blocks.append(TextBlockParam(text=f'[Refusal] {part.refusal}', type='text'))
+
+			# Add tool use blocks if present
+			if message.tool_calls:
+				tool_blocks = AnthropicMessageSerializer._serialize_tool_calls_to_content(
+					message.tool_calls, use_cache=message.cache
+				)
+				blocks.extend(tool_blocks)
+
+			# If no content or tool calls, add empty text block
+			# (Anthropic requires at least one content block)
+			if not blocks:
+				blocks.append(
+					TextBlockParam(
+						text='', type='text', cache_control=AnthropicMessageSerializer._serialize_cache_control(message.cache)
+					)
+				)
+
+			# If caching is enabled or we have multiple blocks, return blocks as-is
+			# Otherwise, simplify single text blocks to plain string
+			if message.cache or len(blocks) > 1:
+				content = blocks
+			else:
+				# Only simplify when no caching and single block
+				single_block = blocks[0]
+				if single_block['type'] == 'text' and not single_block.get('cache_control'):
+					content = single_block['text']
+				else:
+					content = blocks
+
+			return MessageParam(
+				role='assistant',
+				content=content,
+			)
+
+		else:
+			raise ValueError(f'Unknown message type: {type(message)}')
+
+	@staticmethod
+	def _clean_cache_messages(messages: list[NonSystemMessage]) -> list[NonSystemMessage]:
+		"""Clean cache settings so only the last cache=True message remains cached.
+
+		Because of how Claude caching works, only the last cache message matters.
+		This method automatically removes cache=True from all messages except the last one.
+
+		Args:
+			messages: List of non-system messages to clean
+
+		Returns:
+			List of messages with cleaned cache settings
+		"""
+		if not messages:
+			return messages
+
+		# Create a copy to avoid modifying the original
+		cleaned_messages = [msg.model_copy(deep=True) for msg in messages]
+
+		# Find the last message with cache=True
+		last_cache_index = -1
+		for i in range(len(cleaned_messages) - 1, -1, -1):
+			if cleaned_messages[i].cache:
+				last_cache_index = i
+				break
+
+		# If we found a cached message, disable cache for all others
+		if last_cache_index != -1:
+			for i, msg in enumerate(cleaned_messages):
+				if i != last_cache_index and msg.cache:
+					# Set cache to False for all messages except the last cached one
+					msg.cache = False
+
+		return cleaned_messages
+
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> tuple[list[MessageParam], list[TextBlockParam] | str | None]:
+		"""Serialize a list of messages, extracting any system message.
+
+		Returns:
+		    A tuple of (messages, system_message) where system_message is extracted
+		    from any SystemMessage in the list.
+		"""
+		messages = [m.model_copy(deep=True) for m in messages]
+
+		# Separate system messages from normal messages
+		normal_messages: list[NonSystemMessage] = []
+		system_message: SystemMessage | None = None
+
+		for message in messages:
+			if isinstance(message, SystemMessage):
+				system_message = message
+			else:
+				normal_messages.append(message)
+
+		# Clean cache messages so only the last cache=True message remains cached
+		normal_messages = AnthropicMessageSerializer._clean_cache_messages(normal_messages)
+
+		# Serialize normal messages
+		serialized_messages: list[MessageParam] = []
+		for message in normal_messages:
+			serialized_messages.append(AnthropicMessageSerializer.serialize(message))
+
+		# Serialize system message
+		serialized_system_message: list[TextBlockParam] | str | None = None
+		if system_message:
+			serialized_system_message = AnthropicMessageSerializer._serialize_content_to_str(
+				system_message.content, use_cache=system_message.cache
+			)
+
+		return serialized_messages, serialized_system_message

chapter9/browser-use-rpa/browser-use/browser_use/llm/aws/__init__.py

@@ -0,0 +1,36 @@
+from typing import TYPE_CHECKING
+
+# Type stubs for lazy imports
+if TYPE_CHECKING:
+	from browser_use.llm.aws.chat_anthropic import ChatAnthropicBedrock
+	from browser_use.llm.aws.chat_bedrock import ChatAWSBedrock
+
+# Lazy imports mapping for AWS chat models
+_LAZY_IMPORTS = {
+	'ChatAnthropicBedrock': ('browser_use.llm.aws.chat_anthropic', 'ChatAnthropicBedrock'),
+	'ChatAWSBedrock': ('browser_use.llm.aws.chat_bedrock', 'ChatAWSBedrock'),
+}
+
+
+def __getattr__(name: str):
+	"""Lazy import mechanism for AWS chat models."""
+	if name in _LAZY_IMPORTS:
+		module_path, attr_name = _LAZY_IMPORTS[name]
+		try:
+			from importlib import import_module
+
+			module = import_module(module_path)
+			attr = getattr(module, attr_name)
+			# Cache the imported attribute in the module's globals
+			globals()[name] = attr
+			return attr
+		except ImportError as e:
+			raise ImportError(f'Failed to import {name} from {module_path}: {e}') from e
+
+	raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+	'ChatAWSBedrock',
+	'ChatAnthropicBedrock',
+]

chapter9/browser-use-rpa/browser-use/browser_use/llm/aws/chat_anthropic.py

@@ -0,0 +1,242 @@
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, TypeVar, overload
+
+from anthropic import (
+	NOT_GIVEN,
+	APIConnectionError,
+	APIStatusError,
+	AsyncAnthropicBedrock,
+	RateLimitError,
+)
+from anthropic.types import CacheControlEphemeralParam, Message, ToolParam
+from anthropic.types.text_block import TextBlock
+from anthropic.types.tool_choice_tool_param import ToolChoiceToolParam
+from pydantic import BaseModel
+
+from browser_use.llm.anthropic.serializer import AnthropicMessageSerializer
+from browser_use.llm.aws.chat_bedrock import ChatAWSBedrock
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+if TYPE_CHECKING:
+	from boto3.session import Session  # pyright: ignore
+
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatAnthropicBedrock(ChatAWSBedrock):
+	"""
+	AWS Bedrock Anthropic Claude chat model.
+
+	This is a convenience class that provides Claude-specific defaults
+	for the AWS Bedrock service. It inherits all functionality from
+	ChatAWSBedrock but sets Anthropic Claude as the default model.
+	"""
+
+	# Anthropic Claude specific defaults
+	model: str = 'anthropic.claude-3-5-sonnet-20240620-v1:0'
+	max_tokens: int = 8192
+	temperature: float | None = None
+	top_p: float | None = None
+	top_k: int | None = None
+	stop_sequences: list[str] | None = None
+	seed: int | None = None
+
+	# AWS credentials and configuration
+	aws_access_key: str | None = None
+	aws_secret_key: str | None = None
+	aws_session_token: str | None = None
+	aws_region: str | None = None
+	session: 'Session | None' = None
+
+	# Client initialization parameters
+	max_retries: int = 10
+	default_headers: Mapping[str, str] | None = None
+	default_query: Mapping[str, object] | None = None
+
+	@property
+	def provider(self) -> str:
+		return 'anthropic_bedrock'
+
+	def _get_client_params(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary for Bedrock."""
+		client_params: dict[str, Any] = {}
+
+		if self.session:
+			credentials = self.session.get_credentials()
+			client_params.update(
+				{
+					'aws_access_key': credentials.access_key,
+					'aws_secret_key': credentials.secret_key,
+					'aws_session_token': credentials.token,
+					'aws_region': self.session.region_name,
+				}
+			)
+		else:
+			# Use individual credentials
+			if self.aws_access_key:
+				client_params['aws_access_key'] = self.aws_access_key
+			if self.aws_secret_key:
+				client_params['aws_secret_key'] = self.aws_secret_key
+			if self.aws_region:
+				client_params['aws_region'] = self.aws_region
+			if self.aws_session_token:
+				client_params['aws_session_token'] = self.aws_session_token
+
+		# Add optional parameters
+		if self.max_retries:
+			client_params['max_retries'] = self.max_retries
+		if self.default_headers:
+			client_params['default_headers'] = self.default_headers
+		if self.default_query:
+			client_params['default_query'] = self.default_query
+
+		return client_params
+
+	def _get_client_params_for_invoke(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary for invoke."""
+		client_params = {}
+
+		if self.temperature is not None:
+			client_params['temperature'] = self.temperature
+		if self.max_tokens is not None:
+			client_params['max_tokens'] = self.max_tokens
+		if self.top_p is not None:
+			client_params['top_p'] = self.top_p
+		if self.top_k is not None:
+			client_params['top_k'] = self.top_k
+		if self.seed is not None:
+			client_params['seed'] = self.seed
+		if self.stop_sequences is not None:
+			client_params['stop_sequences'] = self.stop_sequences
+
+		return client_params
+
+	def get_client(self) -> AsyncAnthropicBedrock:
+		"""
+		Returns an AsyncAnthropicBedrock client.
+
+		Returns:
+			AsyncAnthropicBedrock: An instance of the AsyncAnthropicBedrock client.
+		"""
+		client_params = self._get_client_params()
+		return AsyncAnthropicBedrock(**client_params)
+
+	@property
+	def name(self) -> str:
+		return str(self.model)
+
+	def _get_usage(self, response: Message) -> ChatInvokeUsage | None:
+		"""Extract usage information from the response."""
+		usage = ChatInvokeUsage(
+			prompt_tokens=response.usage.input_tokens
+			+ (
+				response.usage.cache_read_input_tokens or 0
+			),  # Total tokens in Anthropic are a bit fucked, you have to add cached tokens to the prompt tokens
+			completion_tokens=response.usage.output_tokens,
+			total_tokens=response.usage.input_tokens + response.usage.output_tokens,
+			prompt_cached_tokens=response.usage.cache_read_input_tokens,
+			prompt_cache_creation_tokens=response.usage.cache_creation_input_tokens,
+			prompt_image_tokens=None,
+		)
+		return usage
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: None = None) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T]) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		anthropic_messages, system_prompt = AnthropicMessageSerializer.serialize_messages(messages)
+
+		try:
+			if output_format is None:
+				# Normal completion without structured output
+				response = await self.get_client().messages.create(
+					model=self.model,
+					messages=anthropic_messages,
+					system=system_prompt or NOT_GIVEN,
+					**self._get_client_params_for_invoke(),
+				)
+
+				usage = self._get_usage(response)
+
+				# Extract text from the first content block
+				first_content = response.content[0]
+				if isinstance(first_content, TextBlock):
+					response_text = first_content.text
+				else:
+					# If it's not a text block, convert to string
+					response_text = str(first_content)
+
+				return ChatInvokeCompletion(
+					completion=response_text,
+					usage=usage,
+				)
+
+			else:
+				# Use tool calling for structured output
+				# Create a tool that represents the output format
+				tool_name = output_format.__name__
+				schema = output_format.model_json_schema()
+
+				# Remove title from schema if present (Anthropic doesn't like it in parameters)
+				if 'title' in schema:
+					del schema['title']
+
+				tool = ToolParam(
+					name=tool_name,
+					description=f'Extract information in the format of {tool_name}',
+					input_schema=schema,
+					cache_control=CacheControlEphemeralParam(type='ephemeral'),
+				)
+
+				# Force the model to use this tool
+				tool_choice = ToolChoiceToolParam(type='tool', name=tool_name)
+
+				response = await self.get_client().messages.create(
+					model=self.model,
+					messages=anthropic_messages,
+					tools=[tool],
+					system=system_prompt or NOT_GIVEN,
+					tool_choice=tool_choice,
+					**self._get_client_params_for_invoke(),
+				)
+
+				usage = self._get_usage(response)
+
+				# Extract the tool use block
+				for content_block in response.content:
+					if hasattr(content_block, 'type') and content_block.type == 'tool_use':
+						# Parse the tool input as the structured output
+						try:
+							return ChatInvokeCompletion(completion=output_format.model_validate(content_block.input), usage=usage)
+						except Exception as e:
+							# If validation fails, try to parse it as JSON first
+							if isinstance(content_block.input, str):
+								data = json.loads(content_block.input)
+								return ChatInvokeCompletion(
+									completion=output_format.model_validate(data),
+									usage=usage,
+								)
+							raise e
+
+				# If no tool use block found, raise an error
+				raise ValueError('Expected tool use in response but none found')
+
+		except APIConnectionError as e:
+			raise ModelProviderError(message=e.message, model=self.name) from e
+		except RateLimitError as e:
+			raise ModelRateLimitError(message=e.message, model=self.name) from e
+		except APIStatusError as e:
+			raise ModelProviderError(message=e.message, status_code=e.status_code, model=self.name) from e
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e

chapter9/browser-use-rpa/browser-use/browser_use/llm/aws/chat_bedrock.py

@@ -0,0 +1,289 @@
+import json
+from dataclasses import dataclass
+from os import getenv
+from typing import TYPE_CHECKING, Any, TypeVar, overload
+
+from pydantic import BaseModel
+
+from browser_use.llm.aws.serializer import AWSBedrockMessageSerializer
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+if TYPE_CHECKING:
+	from boto3 import client as AwsClient  # type: ignore
+	from boto3.session import Session  # type: ignore
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatAWSBedrock(BaseChatModel):
+	"""
+	AWS Bedrock chat model supporting multiple providers (Anthropic, Meta, etc.).
+
+	This class provides access to various models via AWS Bedrock,
+	supporting both text generation and structured output via tool calling.
+
+	To use this model, you need to either:
+	1. Set the following environment variables:
+	   - AWS_ACCESS_KEY_ID
+	   - AWS_SECRET_ACCESS_KEY
+	   - AWS_SESSION_TOKEN (only required when using temporary credentials)
+	   - AWS_REGION
+	2. Or provide a boto3 Session object
+	3. Or use AWS SSO authentication
+	"""
+
+	# Model configuration
+	model: str = 'anthropic.claude-3-5-sonnet-20240620-v1:0'
+	max_tokens: int | None = 4096
+	temperature: float | None = None
+	top_p: float | None = None
+	seed: int | None = None
+	stop_sequences: list[str] | None = None
+
+	# AWS credentials and configuration
+	aws_access_key_id: str | None = None
+	aws_secret_access_key: str | None = None
+	aws_session_token: str | None = None
+	aws_region: str | None = None
+	aws_sso_auth: bool = False
+	session: 'Session | None' = None
+
+	# Request parameters
+	request_params: dict[str, Any] | None = None
+
+	# Static
+	@property
+	def provider(self) -> str:
+		return 'aws_bedrock'
+
+	def _get_client(self) -> 'AwsClient':  # type: ignore
+		"""Get the AWS Bedrock client."""
+		try:
+			from boto3 import client as AwsClient  # type: ignore
+		except ImportError:
+			raise ImportError(
+				'`boto3` not installed. Please install using `pip install browser-use[aws] or pip install browser-use[all]`'
+			)
+
+		if self.session:
+			return self.session.client('bedrock-runtime')
+
+		# Get credentials from environment or instance parameters
+		access_key = self.aws_access_key_id or getenv('AWS_ACCESS_KEY_ID')
+		secret_key = self.aws_secret_access_key or getenv('AWS_SECRET_ACCESS_KEY')
+		session_token = self.aws_session_token or getenv('AWS_SESSION_TOKEN')
+		region = self.aws_region or getenv('AWS_REGION') or getenv('AWS_DEFAULT_REGION')
+
+		if self.aws_sso_auth:
+			return AwsClient(service_name='bedrock-runtime', region_name=region)
+		else:
+			if not access_key or not secret_key:
+				raise ModelProviderError(
+					message='AWS credentials not found. Please set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables (and AWS_SESSION_TOKEN if using temporary credentials) or provide a boto3 session.',
+					model=self.name,
+				)
+
+			return AwsClient(
+				service_name='bedrock-runtime',
+				region_name=region,
+				aws_access_key_id=access_key,
+				aws_secret_access_key=secret_key,
+				aws_session_token=session_token,
+			)
+
+	@property
+	def name(self) -> str:
+		return str(self.model)
+
+	def _get_inference_config(self) -> dict[str, Any]:
+		"""Get the inference configuration for the request."""
+		config = {}
+		if self.max_tokens is not None:
+			config['maxTokens'] = self.max_tokens
+		if self.temperature is not None:
+			config['temperature'] = self.temperature
+		if self.top_p is not None:
+			config['topP'] = self.top_p
+		if self.stop_sequences is not None:
+			config['stopSequences'] = self.stop_sequences
+		if self.seed is not None:
+			config['seed'] = self.seed
+		return config
+
+	def _format_tools_for_request(self, output_format: type[BaseModel]) -> list[dict[str, Any]]:
+		"""Format a Pydantic model as a tool for structured output."""
+		schema = output_format.model_json_schema()
+
+		# Convert Pydantic schema to Bedrock tool format
+		properties = {}
+		required = []
+
+		for prop_name, prop_info in schema.get('properties', {}).items():
+			properties[prop_name] = {
+				'type': prop_info.get('type', 'string'),
+				'description': prop_info.get('description', ''),
+			}
+
+		# Add required fields
+		required = schema.get('required', [])
+
+		return [
+			{
+				'toolSpec': {
+					'name': f'extract_{output_format.__name__.lower()}',
+					'description': f'Extract information in the format of {output_format.__name__}',
+					'inputSchema': {'json': {'type': 'object', 'properties': properties, 'required': required}},
+				}
+			}
+		]
+
+	def _get_usage(self, response: dict[str, Any]) -> ChatInvokeUsage | None:
+		"""Extract usage information from the response."""
+		if 'usage' not in response:
+			return None
+
+		usage_data = response['usage']
+		return ChatInvokeUsage(
+			prompt_tokens=usage_data.get('inputTokens', 0),
+			completion_tokens=usage_data.get('outputTokens', 0),
+			total_tokens=usage_data.get('totalTokens', 0),
+			prompt_cached_tokens=None,  # Bedrock doesn't provide this
+			prompt_cache_creation_tokens=None,
+			prompt_image_tokens=None,
+		)
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: None = None) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T]) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		"""
+		Invoke the AWS Bedrock model with the given messages.
+
+		Args:
+			messages: List of chat messages
+			output_format: Optional Pydantic model class for structured output
+
+		Returns:
+			Either a string response or an instance of output_format
+		"""
+		try:
+			from botocore.exceptions import ClientError  # type: ignore
+		except ImportError:
+			raise ImportError(
+				'`boto3` not installed. Please install using `pip install browser-use[aws] or pip install browser-use[all]`'
+			)
+
+		bedrock_messages, system_message = AWSBedrockMessageSerializer.serialize_messages(messages)
+
+		try:
+			# Prepare the request body
+			body: dict[str, Any] = {}
+
+			if system_message:
+				body['system'] = system_message
+
+			inference_config = self._get_inference_config()
+			if inference_config:
+				body['inferenceConfig'] = inference_config
+
+			# Handle structured output via tool calling
+			if output_format is not None:
+				tools = self._format_tools_for_request(output_format)
+				body['toolConfig'] = {'tools': tools}
+
+			# Add any additional request parameters
+			if self.request_params:
+				body.update(self.request_params)
+
+			# Filter out None values
+			body = {k: v for k, v in body.items() if v is not None}
+
+			# Make the API call
+			client = self._get_client()
+			response = client.converse(modelId=self.model, messages=bedrock_messages, **body)
+
+			usage = self._get_usage(response)
+
+			# Extract the response content
+			if 'output' in response and 'message' in response['output']:
+				message = response['output']['message']
+				content = message.get('content', [])
+
+				if output_format is None:
+					# Return text response
+					text_content = []
+					for item in content:
+						if 'text' in item:
+							text_content.append(item['text'])
+
+					response_text = '\n'.join(text_content) if text_content else ''
+					return ChatInvokeCompletion(
+						completion=response_text,
+						usage=usage,
+					)
+				else:
+					# Handle structured output from tool calls
+					for item in content:
+						if 'toolUse' in item:
+							tool_use = item['toolUse']
+							tool_input = tool_use.get('input', {})
+
+							try:
+								# Validate and return the structured output
+								return ChatInvokeCompletion(
+									completion=output_format.model_validate(tool_input),
+									usage=usage,
+								)
+							except Exception as e:
+								# If validation fails, try to parse as JSON first
+								if isinstance(tool_input, str):
+									try:
+										data = json.loads(tool_input)
+										return ChatInvokeCompletion(
+											completion=output_format.model_validate(data),
+											usage=usage,
+										)
+									except json.JSONDecodeError:
+										pass
+								raise ModelProviderError(
+									message=f'Failed to validate structured output: {str(e)}',
+									model=self.name,
+								) from e
+
+					# If no tool use found but output_format was requested
+					raise ModelProviderError(
+						message='Expected structured output but no tool use found in response',
+						model=self.name,
+					)
+
+			# If no valid content found
+			if output_format is None:
+				return ChatInvokeCompletion(
+					completion='',
+					usage=usage,
+				)
+			else:
+				raise ModelProviderError(
+					message='No valid content found in response',
+					model=self.name,
+				)
+
+		except ClientError as e:
+			error_code = e.response.get('Error', {}).get('Code', 'Unknown')
+			error_message = e.response.get('Error', {}).get('Message', str(e))
+
+			if error_code in ['ThrottlingException', 'TooManyRequestsException']:
+				raise ModelRateLimitError(message=error_message, model=self.name) from e
+			else:
+				raise ModelProviderError(message=error_message, model=self.name) from e
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e

chapter9/browser-use-rpa/browser-use/browser_use/llm/aws/serializer.py

@@ -0,0 +1,257 @@
+import base64
+import json
+import re
+from typing import Any, overload
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartRefusalParam,
+	ContentPartTextParam,
+	SystemMessage,
+	ToolCall,
+	UserMessage,
+)
+
+
+class AWSBedrockMessageSerializer:
+	"""Serializer for converting between custom message types and AWS Bedrock message format."""
+
+	@staticmethod
+	def _is_base64_image(url: str) -> bool:
+		"""Check if the URL is a base64 encoded image."""
+		return url.startswith('data:image/')
+
+	@staticmethod
+	def _is_url_image(url: str) -> bool:
+		"""Check if the URL is a regular HTTP/HTTPS image URL."""
+		return url.startswith(('http://', 'https://')) and any(
+			url.lower().endswith(ext) for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp']
+		)
+
+	@staticmethod
+	def _parse_base64_url(url: str) -> tuple[str, bytes]:
+		"""Parse a base64 data URL to extract format and raw bytes."""
+		# Format: data:image/jpeg;base64,<data>
+		if not url.startswith('data:'):
+			raise ValueError(f'Invalid base64 URL: {url}')
+
+		header, data = url.split(',', 1)
+
+		# Extract format from mime type
+		mime_match = re.search(r'image/(\w+)', header)
+		if mime_match:
+			format_name = mime_match.group(1).lower()
+			# Map common formats
+			format_mapping = {'jpg': 'jpeg', 'jpeg': 'jpeg', 'png': 'png', 'gif': 'gif', 'webp': 'webp'}
+			image_format = format_mapping.get(format_name, 'jpeg')
+		else:
+			image_format = 'jpeg'  # Default format
+
+		# Decode base64 data
+		try:
+			image_bytes = base64.b64decode(data)
+		except Exception as e:
+			raise ValueError(f'Failed to decode base64 image data: {e}')
+
+		return image_format, image_bytes
+
+	@staticmethod
+	def _download_and_convert_image(url: str) -> tuple[str, bytes]:
+		"""Download an image from URL and convert to base64 bytes."""
+		try:
+			import httpx
+		except ImportError:
+			raise ImportError('httpx not available. Please install it to use URL images with AWS Bedrock.')
+
+		try:
+			response = httpx.get(url, timeout=30)
+			response.raise_for_status()
+
+			# Detect format from content type or URL
+			content_type = response.headers.get('content-type', '').lower()
+			if 'jpeg' in content_type or url.lower().endswith(('.jpg', '.jpeg')):
+				image_format = 'jpeg'
+			elif 'png' in content_type or url.lower().endswith('.png'):
+				image_format = 'png'
+			elif 'gif' in content_type or url.lower().endswith('.gif'):
+				image_format = 'gif'
+			elif 'webp' in content_type or url.lower().endswith('.webp'):
+				image_format = 'webp'
+			else:
+				image_format = 'jpeg'  # Default format
+
+			return image_format, response.content
+
+		except Exception as e:
+			raise ValueError(f'Failed to download image from {url}: {e}')
+
+	@staticmethod
+	def _serialize_content_part_text(part: ContentPartTextParam) -> dict[str, Any]:
+		"""Convert a text content part to AWS Bedrock format."""
+		return {'text': part.text}
+
+	@staticmethod
+	def _serialize_content_part_image(part: ContentPartImageParam) -> dict[str, Any]:
+		"""Convert an image content part to AWS Bedrock format."""
+		url = part.image_url.url
+
+		if AWSBedrockMessageSerializer._is_base64_image(url):
+			# Handle base64 encoded images
+			image_format, image_bytes = AWSBedrockMessageSerializer._parse_base64_url(url)
+		elif AWSBedrockMessageSerializer._is_url_image(url):
+			# Download and convert URL images
+			image_format, image_bytes = AWSBedrockMessageSerializer._download_and_convert_image(url)
+		else:
+			raise ValueError(f'Unsupported image URL format: {url}')
+
+		return {
+			'image': {
+				'format': image_format,
+				'source': {
+					'bytes': image_bytes,
+				},
+			}
+		}
+
+	@staticmethod
+	def _serialize_user_content(
+		content: str | list[ContentPartTextParam | ContentPartImageParam],
+	) -> list[dict[str, Any]]:
+		"""Serialize content for user messages."""
+		if isinstance(content, str):
+			return [{'text': content}]
+
+		content_blocks: list[dict[str, Any]] = []
+		for part in content:
+			if part.type == 'text':
+				content_blocks.append(AWSBedrockMessageSerializer._serialize_content_part_text(part))
+			elif part.type == 'image_url':
+				content_blocks.append(AWSBedrockMessageSerializer._serialize_content_part_image(part))
+
+		return content_blocks
+
+	@staticmethod
+	def _serialize_system_content(
+		content: str | list[ContentPartTextParam],
+	) -> list[dict[str, Any]]:
+		"""Serialize content for system messages."""
+		if isinstance(content, str):
+			return [{'text': content}]
+
+		content_blocks: list[dict[str, Any]] = []
+		for part in content:
+			if part.type == 'text':
+				content_blocks.append(AWSBedrockMessageSerializer._serialize_content_part_text(part))
+
+		return content_blocks
+
+	@staticmethod
+	def _serialize_assistant_content(
+		content: str | list[ContentPartTextParam | ContentPartRefusalParam] | None,
+	) -> list[dict[str, Any]]:
+		"""Serialize content for assistant messages."""
+		if content is None:
+			return []
+		if isinstance(content, str):
+			return [{'text': content}]
+
+		content_blocks: list[dict[str, Any]] = []
+		for part in content:
+			if part.type == 'text':
+				content_blocks.append(AWSBedrockMessageSerializer._serialize_content_part_text(part))
+			# Skip refusal content parts - AWS Bedrock doesn't need them
+
+		return content_blocks
+
+	@staticmethod
+	def _serialize_tool_call(tool_call: ToolCall) -> dict[str, Any]:
+		"""Convert a tool call to AWS Bedrock format."""
+		try:
+			arguments = json.loads(tool_call.function.arguments)
+		except json.JSONDecodeError:
+			# If arguments aren't valid JSON, wrap them
+			arguments = {'arguments': tool_call.function.arguments}
+
+		return {
+			'toolUse': {
+				'toolUseId': tool_call.id,
+				'name': tool_call.function.name,
+				'input': arguments,
+			}
+		}
+
+	# region - Serialize overloads
+	@overload
+	@staticmethod
+	def serialize(message: UserMessage) -> dict[str, Any]: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: SystemMessage) -> SystemMessage: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: AssistantMessage) -> dict[str, Any]: ...
+
+	@staticmethod
+	def serialize(message: BaseMessage) -> dict[str, Any] | SystemMessage:
+		"""Serialize a custom message to AWS Bedrock format."""
+
+		if isinstance(message, UserMessage):
+			return {
+				'role': 'user',
+				'content': AWSBedrockMessageSerializer._serialize_user_content(message.content),
+			}
+
+		elif isinstance(message, SystemMessage):
+			# System messages are handled separately in AWS Bedrock
+			return message
+
+		elif isinstance(message, AssistantMessage):
+			content_blocks: list[dict[str, Any]] = []
+
+			# Add content blocks if present
+			if message.content is not None:
+				content_blocks.extend(AWSBedrockMessageSerializer._serialize_assistant_content(message.content))
+
+			# Add tool use blocks if present
+			if message.tool_calls:
+				for tool_call in message.tool_calls:
+					content_blocks.append(AWSBedrockMessageSerializer._serialize_tool_call(tool_call))
+
+			# AWS Bedrock requires at least one content block
+			if not content_blocks:
+				content_blocks = [{'text': ''}]
+
+			return {
+				'role': 'assistant',
+				'content': content_blocks,
+			}
+
+		else:
+			raise ValueError(f'Unknown message type: {type(message)}')
+
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> tuple[list[dict[str, Any]], list[dict[str, Any]] | None]:
+		"""
+		Serialize a list of messages, extracting any system message.
+
+		Returns:
+			Tuple of (bedrock_messages, system_message) where system_message is extracted
+			from any SystemMessage in the list.
+		"""
+		bedrock_messages: list[dict[str, Any]] = []
+		system_message: list[dict[str, Any]] | None = None
+
+		for message in messages:
+			if isinstance(message, SystemMessage):
+				# Extract system message content
+				system_message = AWSBedrockMessageSerializer._serialize_system_content(message.content)
+			else:
+				# Serialize and add to regular messages
+				serialized = AWSBedrockMessageSerializer.serialize(message)
+				bedrock_messages.append(serialized)
+
+		return bedrock_messages, system_message

chapter9/browser-use-rpa/browser-use/browser_use/llm/azure/chat.py

@@ -0,0 +1,91 @@
+import os
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+from openai import AsyncAzureOpenAI as AsyncAzureOpenAIClient
+from openai.types.shared import ChatModel
+
+from browser_use.llm.openai.like import ChatOpenAILike
+
+
+@dataclass
+class ChatAzureOpenAI(ChatOpenAILike):
+	"""
+	A class for to interact with any provider using the OpenAI API schema.
+
+	Args:
+	    model (str): The name of the OpenAI model to use. Defaults to "not-provided".
+	    api_key (Optional[str]): The API key to use. Defaults to "not-provided".
+	"""
+
+	# Model configuration
+	model: str | ChatModel
+
+	# Client initialization parameters
+	api_key: str | None = None
+	api_version: str | None = '2024-12-01-preview'
+	azure_endpoint: str | None = None
+	azure_deployment: str | None = None
+	base_url: str | None = None
+	azure_ad_token: str | None = None
+	azure_ad_token_provider: Any | None = None
+
+	default_headers: dict[str, str] | None = None
+	default_query: dict[str, Any] | None = None
+
+	client: AsyncAzureOpenAIClient | None = None
+
+	@property
+	def provider(self) -> str:
+		return 'azure'
+
+	def _get_client_params(self) -> dict[str, Any]:
+		_client_params: dict[str, Any] = {}
+
+		self.api_key = self.api_key or os.getenv('AZURE_OPENAI_API_KEY')
+		self.azure_endpoint = self.azure_endpoint or os.getenv('AZURE_OPENAI_ENDPOINT')
+		self.azure_deployment = self.azure_deployment or os.getenv('AZURE_OPENAI_DEPLOYMENT')
+		params_mapping = {
+			'api_key': self.api_key,
+			'api_version': self.api_version,
+			'organization': self.organization,
+			'azure_endpoint': self.azure_endpoint,
+			'azure_deployment': self.azure_deployment,
+			'base_url': self.base_url,
+			'azure_ad_token': self.azure_ad_token,
+			'azure_ad_token_provider': self.azure_ad_token_provider,
+			'http_client': self.http_client,
+		}
+		if self.default_headers is not None:
+			_client_params['default_headers'] = self.default_headers
+		if self.default_query is not None:
+			_client_params['default_query'] = self.default_query
+
+		_client_params.update({k: v for k, v in params_mapping.items() if v is not None})
+
+		return _client_params
+
+	def get_client(self) -> AsyncAzureOpenAIClient:
+		"""
+		Returns an asynchronous OpenAI client.
+
+		Returns:
+			AsyncAzureOpenAIClient: An instance of the asynchronous OpenAI client.
+		"""
+		if self.client:
+			return self.client
+
+		_client_params: dict[str, Any] = self._get_client_params()
+
+		if self.http_client:
+			_client_params['http_client'] = self.http_client
+		else:
+			# Create a new async HTTP client with custom limits
+			_client_params['http_client'] = httpx.AsyncClient(
+				limits=httpx.Limits(max_connections=20, max_keepalive_connections=6)
+			)
+
+		self.client = AsyncAzureOpenAIClient(**_client_params)
+
+		return self.client

chapter9/browser-use-rpa/browser-use/browser_use/llm/base.py

@@ -0,0 +1,57 @@
+"""
+We have switched all of our code from langchain to openai.types.chat.chat_completion_message_param.
+
+For easier transition we have
+"""
+
+from typing import Any, Protocol, TypeVar, overload, runtime_checkable
+
+from pydantic import BaseModel
+
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.views import ChatInvokeCompletion
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@runtime_checkable
+class BaseChatModel(Protocol):
+	_verified_api_keys: bool = False
+
+	model: str
+
+	@property
+	def provider(self) -> str: ...
+
+	@property
+	def name(self) -> str: ...
+
+	@property
+	def model_name(self) -> str:
+		# for legacy support
+		return self.model
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: None = None) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T]) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]: ...
+
+	@classmethod
+	def __get_pydantic_core_schema__(
+		cls,
+		source_type: type,
+		handler: Any,
+	) -> Any:
+		"""
+		Allow this Protocol to be used in Pydantic models -> very useful to typesafe the agent settings for example.
+		Returns a schema that allows any object (since this is a Protocol).
+		"""
+		from pydantic_core import core_schema
+
+		# Return a schema that accepts any object for Protocol types
+		return core_schema.any_schema()

chapter9/browser-use-rpa/browser-use/browser_use/llm/deepseek/chat.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any, TypeVar, overload
+
+import httpx
+from openai import (
+	APIConnectionError,
+	APIError,
+	APIStatusError,
+	APITimeoutError,
+	AsyncOpenAI,
+	RateLimitError,
+)
+from pydantic import BaseModel
+
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.deepseek.serializer import DeepSeekMessageSerializer
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.llm.views import ChatInvokeCompletion
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatDeepSeek(BaseChatModel):
+	"""DeepSeek /chat/completions wrapper (OpenAI-compatible)."""
+
+	model: str = 'deepseek-chat'
+
+	# Generation parameters
+	max_tokens: int | None = None
+	temperature: float | None = None
+	top_p: float | None = None
+	seed: int | None = None
+
+	# Connection parameters
+	api_key: str | None = None
+	base_url: str | httpx.URL | None = 'https://api.deepseek.com/v1'
+	timeout: float | httpx.Timeout | None = None
+	client_params: dict[str, Any] | None = None
+
+	@property
+	def provider(self) -> str:
+		return 'deepseek'
+
+	def _client(self) -> AsyncOpenAI:
+		return AsyncOpenAI(
+			api_key=self.api_key,
+			base_url=self.base_url,
+			timeout=self.timeout,
+			**(self.client_params or {}),
+		)
+
+	@property
+	def name(self) -> str:
+		return self.model
+
+	@overload
+	async def ainvoke(
+		self,
+		messages: list[BaseMessage],
+		output_format: None = None,
+		tools: list[dict[str, Any]] | None = None,
+		stop: list[str] | None = None,
+	) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(
+		self,
+		messages: list[BaseMessage],
+		output_format: type[T],
+		tools: list[dict[str, Any]] | None = None,
+		stop: list[str] | None = None,
+	) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self,
+		messages: list[BaseMessage],
+		output_format: type[T] | None = None,
+		tools: list[dict[str, Any]] | None = None,
+		stop: list[str] | None = None,
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		"""
+		DeepSeek ainvoke supports:
+		1. Regular text/multi-turn conversation
+		2. Function Calling
+		3. JSON Output (response_format)
+		4. Conversation prefix continuation (beta, prefix, stop)
+		"""
+		client = self._client()
+		ds_messages = DeepSeekMessageSerializer.serialize_messages(messages)
+		common: dict[str, Any] = {}
+
+		if self.temperature is not None:
+			common['temperature'] = self.temperature
+		if self.max_tokens is not None:
+			common['max_tokens'] = self.max_tokens
+		if self.top_p is not None:
+			common['top_p'] = self.top_p
+		if self.seed is not None:
+			common['seed'] = self.seed
+
+		# Beta conversation prefix continuation (see official documentation)
+		if self.base_url and str(self.base_url).endswith('/beta'):
+			# The last assistant message must have prefix
+			if ds_messages and isinstance(ds_messages[-1], dict) and ds_messages[-1].get('role') == 'assistant':
+				ds_messages[-1]['prefix'] = True
+			if stop:
+				common['stop'] = stop
+
+		# ① Regular multi-turn conversation/text output
+		if output_format is None and not tools:
+			try:
+				resp = await client.chat.completions.create(  # type: ignore
+					model=self.model,
+					messages=ds_messages,  # type: ignore
+					**common,
+				)
+				return ChatInvokeCompletion(
+					completion=resp.choices[0].message.content or '',
+					usage=None,
+				)
+			except RateLimitError as e:
+				raise ModelRateLimitError(str(e), model=self.name) from e
+			except (APIError, APIConnectionError, APITimeoutError, APIStatusError) as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+			except Exception as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+
+		# ② Function Calling path (with tools or output_format)
+		if tools or (output_format is not None and hasattr(output_format, 'model_json_schema')):
+			try:
+				call_tools = tools
+				tool_choice = None
+				if output_format is not None and hasattr(output_format, 'model_json_schema'):
+					tool_name = output_format.__name__
+					schema = SchemaOptimizer.create_optimized_json_schema(output_format)
+					schema.pop('title', None)
+					call_tools = [
+						{
+							'type': 'function',
+							'function': {
+								'name': tool_name,
+								'description': f'Return a JSON object of type {tool_name}',
+								'parameters': schema,
+							},
+						}
+					]
+					tool_choice = {'type': 'function', 'function': {'name': tool_name}}
+				resp = await client.chat.completions.create(  # type: ignore
+					model=self.model,
+					messages=ds_messages,  # type: ignore
+					tools=call_tools,  # type: ignore
+					tool_choice=tool_choice,  # type: ignore
+					**common,
+				)
+				msg = resp.choices[0].message
+				if not msg.tool_calls:
+					raise ValueError('Expected tool_calls in response but got none')
+				raw_args = msg.tool_calls[0].function.arguments
+				if isinstance(raw_args, str):
+					parsed = json.loads(raw_args)
+				else:
+					parsed = raw_args
+				# --------- Fix: only use model_validate when output_format is not None ----------
+				if output_format is not None:
+					return ChatInvokeCompletion(
+						completion=output_format.model_validate(parsed),
+						usage=None,
+					)
+				else:
+					# If no output_format, return dict directly
+					return ChatInvokeCompletion(
+						completion=parsed,
+						usage=None,
+					)
+			except RateLimitError as e:
+				raise ModelRateLimitError(str(e), model=self.name) from e
+			except (APIError, APIConnectionError, APITimeoutError, APIStatusError) as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+			except Exception as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+
+		# ③ JSON Output path (official response_format)
+		if output_format is not None and hasattr(output_format, 'model_json_schema'):
+			try:
+				resp = await client.chat.completions.create(  # type: ignore
+					model=self.model,
+					messages=ds_messages,  # type: ignore
+					response_format={'type': 'json_object'},
+					**common,
+				)
+				content = resp.choices[0].message.content
+				if not content:
+					raise ModelProviderError('Empty JSON content in DeepSeek response', model=self.name)
+				parsed = output_format.model_validate_json(content)
+				return ChatInvokeCompletion(
+					completion=parsed,
+					usage=None,
+				)
+			except RateLimitError as e:
+				raise ModelRateLimitError(str(e), model=self.name) from e
+			except (APIError, APIConnectionError, APITimeoutError, APIStatusError) as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+			except Exception as e:
+				raise ModelProviderError(str(e), model=self.name) from e
+
+		raise ModelProviderError('No valid ainvoke execution path for DeepSeek LLM', model=self.name)

chapter9/browser-use-rpa/browser-use/browser_use/llm/deepseek/serializer.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import json
+from typing import Any, overload
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	SystemMessage,
+	ToolCall,
+	UserMessage,
+)
+
+MessageDict = dict[str, Any]
+
+
+class DeepSeekMessageSerializer:
+	"""Serializer for converting browser-use messages to DeepSeek messages."""
+
+	# -------- content 处理 --------------------------------------------------
+	@staticmethod
+	def _serialize_text_part(part: ContentPartTextParam) -> str:
+		return part.text
+
+	@staticmethod
+	def _serialize_image_part(part: ContentPartImageParam) -> dict[str, Any]:
+		url = part.image_url.url
+		if url.startswith('data:'):
+			return {'type': 'image_url', 'image_url': {'url': url}}
+		return {'type': 'image_url', 'image_url': {'url': url}}
+
+	@staticmethod
+	def _serialize_content(content: Any) -> str | list[dict[str, Any]]:
+		if content is None:
+			return ''
+		if isinstance(content, str):
+			return content
+		serialized: list[dict[str, Any]] = []
+		for part in content:
+			if part.type == 'text':
+				serialized.append({'type': 'text', 'text': DeepSeekMessageSerializer._serialize_text_part(part)})
+			elif part.type == 'image_url':
+				serialized.append(DeepSeekMessageSerializer._serialize_image_part(part))
+			elif part.type == 'refusal':
+				serialized.append({'type': 'text', 'text': f'[Refusal] {part.refusal}'})
+		return serialized
+
+	# -------- Tool-call 处理 -------------------------------------------------
+	@staticmethod
+	def _serialize_tool_calls(tool_calls: list[ToolCall]) -> list[dict[str, Any]]:
+		deepseek_tool_calls: list[dict[str, Any]] = []
+		for tc in tool_calls:
+			try:
+				arguments = json.loads(tc.function.arguments)
+			except json.JSONDecodeError:
+				arguments = {'arguments': tc.function.arguments}
+			deepseek_tool_calls.append(
+				{
+					'id': tc.id,
+					'type': 'function',
+					'function': {
+						'name': tc.function.name,
+						'arguments': arguments,
+					},
+				}
+			)
+		return deepseek_tool_calls
+
+	# -------- 单条消息序列化 -------------------------------------------------
+	@overload
+	@staticmethod
+	def serialize(message: UserMessage) -> MessageDict: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: SystemMessage) -> MessageDict: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: AssistantMessage) -> MessageDict: ...
+
+	@staticmethod
+	def serialize(message: BaseMessage) -> MessageDict:
+		if isinstance(message, UserMessage):
+			return {
+				'role': 'user',
+				'content': DeepSeekMessageSerializer._serialize_content(message.content),
+			}
+		if isinstance(message, SystemMessage):
+			return {
+				'role': 'system',
+				'content': DeepSeekMessageSerializer._serialize_content(message.content),
+			}
+		if isinstance(message, AssistantMessage):
+			msg: MessageDict = {
+				'role': 'assistant',
+				'content': DeepSeekMessageSerializer._serialize_content(message.content),
+			}
+			if message.tool_calls:
+				msg['tool_calls'] = DeepSeekMessageSerializer._serialize_tool_calls(message.tool_calls)
+			return msg
+		raise ValueError(f'Unknown message type: {type(message)}')
+
+	# -------- 列表序列化 -----------------------------------------------------
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> list[MessageDict]:
+		return [DeepSeekMessageSerializer.serialize(m) for m in messages]

chapter9/browser-use-rpa/browser-use/browser_use/llm/exceptions.py

@@ -0,0 +1,27 @@
+class ModelError(Exception):
+	pass
+
+
+class ModelProviderError(ModelError):
+	"""Exception raised when a model provider returns an error."""
+
+	def __init__(
+		self,
+		message: str,
+		status_code: int = 502,
+		model: str | None = None,
+	):
+		super().__init__(message, status_code)
+		self.model = model
+
+
+class ModelRateLimitError(ModelProviderError):
+	"""Exception raised when a model provider returns a rate limit error."""
+
+	def __init__(
+		self,
+		message: str,
+		status_code: int = 429,
+		model: str | None = None,
+	):
+		super().__init__(message, status_code, model)

chapter9/browser-use-rpa/browser-use/browser_use/llm/google/__init__.py

@@ -0,0 +1,3 @@
+from browser_use.llm.google.chat import ChatGoogle
+
+__all__ = ['ChatGoogle']

chapter9/browser-use-rpa/browser-use/browser_use/llm/google/chat.py

@@ -0,0 +1,506 @@
+import asyncio
+import json
+import logging
+import time
+from dataclasses import dataclass
+from typing import Any, Literal, TypeVar, overload
+
+from google import genai
+from google.auth.credentials import Credentials
+from google.genai import types
+from google.genai.types import MediaModality
+from pydantic import BaseModel
+
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.exceptions import ModelProviderError
+from browser_use.llm.google.serializer import GoogleMessageSerializer
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+T = TypeVar('T', bound=BaseModel)
+
+
+VerifiedGeminiModels = Literal[
+	'gemini-2.0-flash',
+	'gemini-2.0-flash-exp',
+	'gemini-2.0-flash-lite-preview-02-05',
+	'Gemini-2.0-exp',
+	'gemini-2.5-flash',
+	'gemini-2.5-flash-lite',
+	'gemini-2.5-pro',
+	'gemma-3-27b-it',
+	'gemma-3-4b',
+	'gemma-3-12b',
+	'gemma-3n-e2b',
+	'gemma-3n-e4b',
+]
+
+
+@dataclass
+class ChatGoogle(BaseChatModel):
+	"""
+	A wrapper around Google's Gemini chat model using the genai client.
+
+	This class accepts all genai.Client parameters while adding model,
+	temperature, and config parameters for the LLM interface.
+
+	Args:
+		model: The Gemini model to use
+		temperature: Temperature for response generation
+		config: Additional configuration parameters to pass to generate_content
+			(e.g., tools, safety_settings, etc.).
+		api_key: Google API key
+		vertexai: Whether to use Vertex AI
+		credentials: Google credentials object
+		project: Google Cloud project ID
+		location: Google Cloud location
+		http_options: HTTP options for the client
+		include_system_in_user: If True, system messages are included in the first user message
+		supports_structured_output: If True, uses native JSON mode; if False, uses prompt-based fallback
+
+	Example:
+		from google.genai import types
+
+		llm = ChatGoogle(
+			model='gemini-2.0-flash-exp',
+			config={
+				'tools': [types.Tool(code_execution=types.ToolCodeExecution())]
+			}
+		)
+	"""
+
+	# Model configuration
+	model: VerifiedGeminiModels | str
+	temperature: float | None = 0.2
+	top_p: float | None = None
+	seed: int | None = None
+	thinking_budget: int | None = None
+	max_output_tokens: int | None = 4096
+	config: types.GenerateContentConfigDict | None = None
+	include_system_in_user: bool = False
+	supports_structured_output: bool = True  # New flag
+
+	# Client initialization parameters
+	api_key: str | None = None
+	vertexai: bool | None = None
+	credentials: Credentials | None = None
+	project: str | None = None
+	location: str | None = None
+	http_options: types.HttpOptions | types.HttpOptionsDict | None = None
+
+	# Static
+	@property
+	def provider(self) -> str:
+		return 'google'
+
+	@property
+	def logger(self) -> logging.Logger:
+		"""Get logger for this chat instance"""
+		return logging.getLogger(f'browser_use.llm.google.{self.model}')
+
+	def _get_client_params(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary."""
+		# Define base client params
+		base_params = {
+			'api_key': self.api_key,
+			'vertexai': self.vertexai,
+			'credentials': self.credentials,
+			'project': self.project,
+			'location': self.location,
+			'http_options': self.http_options,
+		}
+
+		# Create client_params dict with non-None values
+		client_params = {k: v for k, v in base_params.items() if v is not None}
+
+		return client_params
+
+	def get_client(self) -> genai.Client:
+		"""
+		Returns a genai.Client instance.
+
+		Returns:
+			genai.Client: An instance of the Google genai client.
+		"""
+		client_params = self._get_client_params()
+		return genai.Client(**client_params)
+
+	@property
+	def name(self) -> str:
+		return str(self.model)
+
+	def _get_usage(self, response: types.GenerateContentResponse) -> ChatInvokeUsage | None:
+		usage: ChatInvokeUsage | None = None
+
+		if response.usage_metadata is not None:
+			image_tokens = 0
+			if response.usage_metadata.prompt_tokens_details is not None:
+				image_tokens = sum(
+					detail.token_count or 0
+					for detail in response.usage_metadata.prompt_tokens_details
+					if detail.modality == MediaModality.IMAGE
+				)
+
+			usage = ChatInvokeUsage(
+				prompt_tokens=response.usage_metadata.prompt_token_count or 0,
+				completion_tokens=(response.usage_metadata.candidates_token_count or 0)
+				+ (response.usage_metadata.thoughts_token_count or 0),
+				total_tokens=response.usage_metadata.total_token_count or 0,
+				prompt_cached_tokens=response.usage_metadata.cached_content_token_count,
+				prompt_cache_creation_tokens=None,
+				prompt_image_tokens=image_tokens,
+			)
+
+		return usage
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: None = None) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T]) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		"""
+		Invoke the model with the given messages.
+
+		Args:
+			messages: List of chat messages
+			output_format: Optional Pydantic model class for structured output
+
+		Returns:
+			Either a string response or an instance of output_format
+		"""
+
+		# Serialize messages to Google format with the include_system_in_user flag
+		contents, system_instruction = GoogleMessageSerializer.serialize_messages(
+			messages, include_system_in_user=self.include_system_in_user
+		)
+
+		# Build config dictionary starting with user-provided config
+		config: types.GenerateContentConfigDict = {}
+		if self.config:
+			config = self.config.copy()
+
+		# Apply model-specific configuration (these can override config)
+		if self.temperature is not None:
+			config['temperature'] = self.temperature
+
+		# Add system instruction if present
+		if system_instruction:
+			config['system_instruction'] = system_instruction
+
+		if self.top_p is not None:
+			config['top_p'] = self.top_p
+
+		if self.seed is not None:
+			config['seed'] = self.seed
+
+		if self.thinking_budget is None and 'gemini-2.5-flash' in self.model:
+			self.thinking_budget = 0
+
+		if self.thinking_budget is not None:
+			thinking_config_dict: types.ThinkingConfigDict = {'thinking_budget': self.thinking_budget}
+			config['thinking_config'] = thinking_config_dict
+
+		if self.max_output_tokens is not None:
+			config['max_output_tokens'] = self.max_output_tokens
+
+		async def _make_api_call():
+			start_time = time.time()
+			self.logger.debug(f'🚀 Starting API call to {self.model}')
+
+			try:
+				if output_format is None:
+					# Return string response
+					self.logger.debug('📄 Requesting text response')
+
+					response = await self.get_client().aio.models.generate_content(
+						model=self.model,
+						contents=contents,  # type: ignore
+						config=config,
+					)
+
+					elapsed = time.time() - start_time
+					self.logger.debug(f'✅ Got text response in {elapsed:.2f}s')
+
+					# Handle case where response.text might be None
+					text = response.text or ''
+					if not text:
+						self.logger.warning('⚠️ Empty text response received')
+
+					usage = self._get_usage(response)
+
+					return ChatInvokeCompletion(
+						completion=text,
+						usage=usage,
+					)
+
+				else:
+					# Handle structured output
+					if self.supports_structured_output:
+						# Use native JSON mode
+						self.logger.debug(f'🔧 Requesting structured output for {output_format.__name__}')
+						config['response_mime_type'] = 'application/json'
+						# Convert Pydantic model to Gemini-compatible schema
+						optimized_schema = SchemaOptimizer.create_optimized_json_schema(output_format)
+
+						gemini_schema = self._fix_gemini_schema(optimized_schema)
+						config['response_schema'] = gemini_schema
+
+						response = await self.get_client().aio.models.generate_content(
+							model=self.model,
+							contents=contents,
+							config=config,
+						)
+
+						elapsed = time.time() - start_time
+						self.logger.debug(f'✅ Got structured response in {elapsed:.2f}s')
+
+						usage = self._get_usage(response)
+
+						# Handle case where response.parsed might be None
+						if response.parsed is None:
+							self.logger.debug('📝 Parsing JSON from text response')
+							# When using response_schema, Gemini returns JSON as text
+							if response.text:
+								try:
+									# Handle JSON wrapped in markdown code blocks (common Gemini behavior)
+									text = response.text.strip()
+									if text.startswith('```json') and text.endswith('```'):
+										text = text[7:-3].strip()
+										self.logger.debug('🔧 Stripped ```json``` wrapper from response')
+									elif text.startswith('```') and text.endswith('```'):
+										text = text[3:-3].strip()
+										self.logger.debug('🔧 Stripped ``` wrapper from response')
+
+									# Parse the JSON text and validate with the Pydantic model
+									parsed_data = json.loads(text)
+									return ChatInvokeCompletion(
+										completion=output_format.model_validate(parsed_data),
+										usage=usage,
+									)
+								except (json.JSONDecodeError, ValueError) as e:
+									self.logger.error(f'❌ Failed to parse JSON response: {str(e)}')
+									self.logger.debug(f'Raw response text: {response.text[:200]}...')
+									raise ModelProviderError(
+										message=f'Failed to parse or validate response {response}: {str(e)}',
+										status_code=500,
+										model=self.model,
+									) from e
+							else:
+								self.logger.error('❌ No response text received')
+								raise ModelProviderError(
+									message=f'No response from model {response}',
+									status_code=500,
+									model=self.model,
+								)
+
+						# Ensure we return the correct type
+						if isinstance(response.parsed, output_format):
+							return ChatInvokeCompletion(
+								completion=response.parsed,
+								usage=usage,
+							)
+						else:
+							# If it's not the expected type, try to validate it
+							return ChatInvokeCompletion(
+								completion=output_format.model_validate(response.parsed),
+								usage=usage,
+							)
+					else:
+						# Fallback: Request JSON in the prompt for models without native JSON mode
+						self.logger.debug(f'🔄 Using fallback JSON mode for {output_format.__name__}')
+						# Create a copy of messages to modify
+						modified_messages = [m.model_copy(deep=True) for m in messages]
+
+						# Add JSON instruction to the last message
+						if modified_messages and isinstance(modified_messages[-1].content, str):
+							json_instruction = f'\n\nPlease respond with a valid JSON object that matches this schema: {SchemaOptimizer.create_optimized_json_schema(output_format)}'
+							modified_messages[-1].content += json_instruction
+
+						# Re-serialize with modified messages
+						fallback_contents, fallback_system = GoogleMessageSerializer.serialize_messages(
+							modified_messages, include_system_in_user=self.include_system_in_user
+						)
+
+						# Update config with fallback system instruction if present
+						fallback_config = config.copy()
+						if fallback_system:
+							fallback_config['system_instruction'] = fallback_system
+
+						response = await self.get_client().aio.models.generate_content(
+							model=self.model,
+							contents=fallback_contents,  # type: ignore
+							config=fallback_config,
+						)
+
+						elapsed = time.time() - start_time
+						self.logger.debug(f'✅ Got fallback response in {elapsed:.2f}s')
+
+						usage = self._get_usage(response)
+
+						# Try to extract JSON from the text response
+						if response.text:
+							try:
+								# Try to find JSON in the response
+								text = response.text.strip()
+
+								# Common patterns: JSON wrapped in markdown code blocks
+								if text.startswith('```json') and text.endswith('```'):
+									text = text[7:-3].strip()
+								elif text.startswith('```') and text.endswith('```'):
+									text = text[3:-3].strip()
+
+								# Parse and validate
+								parsed_data = json.loads(text)
+								return ChatInvokeCompletion(
+									completion=output_format.model_validate(parsed_data),
+									usage=usage,
+								)
+							except (json.JSONDecodeError, ValueError) as e:
+								self.logger.error(f'❌ Failed to parse fallback JSON: {str(e)}')
+								self.logger.debug(f'Raw response text: {response.text[:200]}...')
+								raise ModelProviderError(
+									message=f'Model does not support JSON mode and failed to parse JSON from text response: {str(e)}',
+									status_code=500,
+									model=self.model,
+								) from e
+						else:
+							self.logger.error('❌ No response text in fallback mode')
+							raise ModelProviderError(
+								message='No response from model',
+								status_code=500,
+								model=self.model,
+							)
+			except Exception as e:
+				elapsed = time.time() - start_time
+				self.logger.error(f'💥 API call failed after {elapsed:.2f}s: {type(e).__name__}: {e}')
+				# Re-raise the exception
+				raise
+
+		try:
+			# Let Google client handle retries internally with proper connection management
+			self.logger.debug(f'🔄 Making API call to {self.model} (using built-in retry)')
+			return await _make_api_call()
+
+		except Exception as e:
+			# Handle specific Google API errors with enhanced diagnostics
+			error_message = str(e)
+			status_code: int | None = None
+
+			# Enhanced timeout error handling
+			if 'timeout' in error_message.lower() or 'cancelled' in error_message.lower():
+				if isinstance(e, asyncio.CancelledError) or 'CancelledError' in str(type(e)):
+					enhanced_message = 'Gemini API request was cancelled (likely timeout). '
+					enhanced_message += 'This suggests the API is taking too long to respond. '
+					enhanced_message += (
+						'Consider: 1) Reducing input size, 2) Using a different model, 3) Checking network connectivity.'
+					)
+					error_message = enhanced_message
+					status_code = 504  # Gateway timeout
+					self.logger.error(f'🕐 Timeout diagnosis: Model: {self.model}')
+				else:
+					status_code = 408  # Request timeout
+			# Check if this is a rate limit error
+			elif any(
+				indicator in error_message.lower()
+				for indicator in ['rate limit', 'resource exhausted', 'quota exceeded', 'too many requests', '429']
+			):
+				status_code = 429
+			elif any(
+				indicator in error_message.lower()
+				for indicator in ['service unavailable', 'internal server error', 'bad gateway', '503', '502', '500']
+			):
+				status_code = 503
+
+			# Try to extract status code if available
+			if hasattr(e, 'response'):
+				response_obj = getattr(e, 'response', None)
+				if response_obj and hasattr(response_obj, 'status_code'):
+					status_code = getattr(response_obj, 'status_code', None)
+
+			raise ModelProviderError(
+				message=error_message,
+				status_code=status_code or 502,  # Use default if None
+				model=self.name,
+			) from e
+
+	def _fix_gemini_schema(self, schema: dict[str, Any]) -> dict[str, Any]:
+		"""
+		Convert a Pydantic model to a Gemini-compatible schema.
+
+		This function removes unsupported properties like 'additionalProperties' and resolves
+		$ref references that Gemini doesn't support.
+		"""
+
+		# Handle $defs and $ref resolution
+		if '$defs' in schema:
+			defs = schema.pop('$defs')
+
+			def resolve_refs(obj: Any) -> Any:
+				if isinstance(obj, dict):
+					if '$ref' in obj:
+						ref = obj.pop('$ref')
+						ref_name = ref.split('/')[-1]
+						if ref_name in defs:
+							# Replace the reference with the actual definition
+							resolved = defs[ref_name].copy()
+							# Merge any additional properties from the reference
+							for key, value in obj.items():
+								if key != '$ref':
+									resolved[key] = value
+							return resolve_refs(resolved)
+						return obj
+					else:
+						# Recursively process all dictionary values
+						return {k: resolve_refs(v) for k, v in obj.items()}
+				elif isinstance(obj, list):
+					return [resolve_refs(item) for item in obj]
+				return obj
+
+			schema = resolve_refs(schema)
+
+		# Remove unsupported properties
+		def clean_schema(obj: Any) -> Any:
+			if isinstance(obj, dict):
+				# Remove unsupported properties
+				cleaned = {}
+				for key, value in obj.items():
+					if key not in ['additionalProperties', 'title', 'default']:
+						cleaned_value = clean_schema(value)
+						# Handle empty object properties - Gemini doesn't allow empty OBJECT types
+						if (
+							key == 'properties'
+							and isinstance(cleaned_value, dict)
+							and len(cleaned_value) == 0
+							and isinstance(obj.get('type', ''), str)
+							and obj.get('type', '').upper() == 'OBJECT'
+						):
+							# Convert empty object to have at least one property
+							cleaned['properties'] = {'_placeholder': {'type': 'string'}}
+						else:
+							cleaned[key] = cleaned_value
+
+				# If this is an object type with empty properties, add a placeholder
+				if (
+					isinstance(cleaned.get('type', ''), str)
+					and cleaned.get('type', '').upper() == 'OBJECT'
+					and 'properties' in cleaned
+					and isinstance(cleaned['properties'], dict)
+					and len(cleaned['properties']) == 0
+				):
+					cleaned['properties'] = {'_placeholder': {'type': 'string'}}
+
+				# Also remove 'title' from the required list if it exists
+				if 'required' in cleaned and isinstance(cleaned.get('required'), list):
+					cleaned['required'] = [p for p in cleaned['required'] if p != 'title']
+
+				return cleaned
+			elif isinstance(obj, list):
+				return [clean_schema(item) for item in obj]
+			return obj
+
+		return clean_schema(schema)

chapter9/browser-use-rpa/browser-use/browser_use/llm/google/serializer.py

@@ -0,0 +1,120 @@
+import base64
+
+from google.genai.types import Content, ContentListUnion, Part
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	SystemMessage,
+	UserMessage,
+)
+
+
+class GoogleMessageSerializer:
+	"""Serializer for converting messages to Google Gemini format."""
+
+	@staticmethod
+	def serialize_messages(
+		messages: list[BaseMessage], include_system_in_user: bool = False
+	) -> tuple[ContentListUnion, str | None]:
+		"""
+		Convert a list of BaseMessages to Google format, extracting system message.
+
+		Google handles system instructions separately from the conversation, so we need to:
+		1. Extract any system messages and return them separately as a string (or include in first user message if flag is set)
+		2. Convert the remaining messages to Content objects
+
+		Args:
+		    messages: List of messages to convert
+		    include_system_in_user: If True, system/developer messages are prepended to the first user message
+
+		Returns:
+		    A tuple of (formatted_messages, system_message) where:
+		    - formatted_messages: List of Content objects for the conversation
+		    - system_message: System instruction string or None
+		"""
+
+		messages = [m.model_copy(deep=True) for m in messages]
+
+		formatted_messages: ContentListUnion = []
+		system_message: str | None = None
+		system_parts: list[str] = []
+
+		for i, message in enumerate(messages):
+			role = message.role if hasattr(message, 'role') else None
+
+			# Handle system/developer messages
+			if isinstance(message, SystemMessage) or role in ['system', 'developer']:
+				# Extract system message content as string
+				if isinstance(message.content, str):
+					if include_system_in_user:
+						system_parts.append(message.content)
+					else:
+						system_message = message.content
+				elif message.content is not None:
+					# Handle Iterable of content parts
+					parts = []
+					for part in message.content:
+						if part.type == 'text':
+							parts.append(part.text)
+					combined_text = '\n'.join(parts)
+					if include_system_in_user:
+						system_parts.append(combined_text)
+					else:
+						system_message = combined_text
+				continue
+
+			# Determine the role for non-system messages
+			if isinstance(message, UserMessage):
+				role = 'user'
+			elif isinstance(message, AssistantMessage):
+				role = 'model'
+			else:
+				# Default to user for any unknown message types
+				role = 'user'
+
+			# Initialize message parts
+			message_parts: list[Part] = []
+
+			# If this is the first user message and we have system parts, prepend them
+			if include_system_in_user and system_parts and role == 'user' and not formatted_messages:
+				system_text = '\n\n'.join(system_parts)
+				if isinstance(message.content, str):
+					message_parts.append(Part.from_text(text=f'{system_text}\n\n{message.content}'))
+				else:
+					# Add system text as the first part
+					message_parts.append(Part.from_text(text=system_text))
+				system_parts = []  # Clear after using
+			else:
+				# Extract content and create parts normally
+				if isinstance(message.content, str):
+					# Regular text content
+					message_parts = [Part.from_text(text=message.content)]
+				elif message.content is not None:
+					# Handle Iterable of content parts
+					for part in message.content:
+						if part.type == 'text':
+							message_parts.append(Part.from_text(text=part.text))
+						elif part.type == 'refusal':
+							message_parts.append(Part.from_text(text=f'[Refusal] {part.refusal}'))
+						elif part.type == 'image_url':
+							# Handle images
+							url = part.image_url.url
+
+							# Format: data:image/png;base64,<data>
+							header, data = url.split(',', 1)
+							# Decode base64 to bytes
+							image_bytes = base64.b64decode(data)
+
+							# Add image part
+							image_part = Part.from_bytes(data=image_bytes, mime_type='image/png')
+
+							message_parts.append(image_part)
+
+			# Create the Content object
+			if message_parts:
+				final_message = Content(role=role, parts=message_parts)
+				# for some reason, the type checker is not able to infer the type of formatted_messages
+				formatted_messages.append(final_message)  # type: ignore
+
+		return formatted_messages, system_message

chapter9/browser-use-rpa/browser-use/browser_use/llm/groq/chat.py

@@ -0,0 +1,229 @@
+import logging
+from dataclasses import dataclass
+from typing import Literal, TypeVar, overload
+
+from groq import (
+	APIError,
+	APIResponseValidationError,
+	APIStatusError,
+	AsyncGroq,
+	NotGiven,
+	RateLimitError,
+	Timeout,
+)
+from groq.types.chat import ChatCompletion, ChatCompletionToolChoiceOptionParam, ChatCompletionToolParam
+from groq.types.chat.completion_create_params import (
+	ResponseFormatResponseFormatJsonSchema,
+	ResponseFormatResponseFormatJsonSchemaJsonSchema,
+)
+from httpx import URL
+from pydantic import BaseModel
+
+from browser_use.llm.base import BaseChatModel, ChatInvokeCompletion
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.groq.parser import try_parse_groq_failed_generation
+from browser_use.llm.groq.serializer import GroqMessageSerializer
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.llm.views import ChatInvokeUsage
+
+GroqVerifiedModels = Literal[
+	'meta-llama/llama-4-maverick-17b-128e-instruct',
+	'meta-llama/llama-4-scout-17b-16e-instruct',
+	'qwen/qwen3-32b',
+	'moonshotai/kimi-k2-instruct',
+	'openai/gpt-oss-20b',
+	'openai/gpt-oss-120b',
+]
+
+JsonSchemaModels = [
+	'meta-llama/llama-4-maverick-17b-128e-instruct',
+	'meta-llama/llama-4-scout-17b-16e-instruct',
+	'openai/gpt-oss-20b',
+	'openai/gpt-oss-120b',
+]
+
+ToolCallingModels = [
+	'moonshotai/kimi-k2-instruct',
+]
+
+T = TypeVar('T', bound=BaseModel)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ChatGroq(BaseChatModel):
+	"""
+	A wrapper around AsyncGroq that implements the BaseLLM protocol.
+	"""
+
+	# Model configuration
+	model: GroqVerifiedModels | str
+
+	# Model params
+	temperature: float | None = None
+	service_tier: Literal['auto', 'on_demand', 'flex'] | None = None
+	top_p: float | None = None
+	seed: int | None = None
+
+	# Client initialization parameters
+	api_key: str | None = None
+	base_url: str | URL | None = None
+	timeout: float | Timeout | NotGiven | None = None
+	max_retries: int = 10  # Increase default retries for automation reliability
+
+	def get_client(self) -> AsyncGroq:
+		return AsyncGroq(api_key=self.api_key, base_url=self.base_url, timeout=self.timeout, max_retries=self.max_retries)
+
+	@property
+	def provider(self) -> str:
+		return 'groq'
+
+	@property
+	def name(self) -> str:
+		return str(self.model)
+
+	def _get_usage(self, response: ChatCompletion) -> ChatInvokeUsage | None:
+		usage = (
+			ChatInvokeUsage(
+				prompt_tokens=response.usage.prompt_tokens,
+				completion_tokens=response.usage.completion_tokens,
+				total_tokens=response.usage.total_tokens,
+				prompt_cached_tokens=None,  # Groq doesn't support cached tokens
+				prompt_cache_creation_tokens=None,
+				prompt_image_tokens=None,
+			)
+			if response.usage is not None
+			else None
+		)
+		return usage
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: None = None) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T]) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		groq_messages = GroqMessageSerializer.serialize_messages(messages)
+
+		try:
+			if output_format is None:
+				return await self._invoke_regular_completion(groq_messages)
+			else:
+				return await self._invoke_structured_output(groq_messages, output_format)
+
+		except RateLimitError as e:
+			raise ModelRateLimitError(message=e.response.text, status_code=e.response.status_code, model=self.name) from e
+
+		except APIResponseValidationError as e:
+			raise ModelProviderError(message=e.response.text, status_code=e.response.status_code, model=self.name) from e
+
+		except APIStatusError as e:
+			if output_format is None:
+				raise ModelProviderError(message=e.response.text, status_code=e.response.status_code, model=self.name) from e
+			else:
+				try:
+					logger.debug(f'Groq failed generation: {e.response.text}; fallback to manual parsing')
+
+					parsed_response = try_parse_groq_failed_generation(e, output_format)
+
+					logger.debug('Manual error parsing successful ✅')
+
+					return ChatInvokeCompletion(
+						completion=parsed_response,
+						usage=None,  # because this is a hacky way to get the outputs
+						# TODO: @groq needs to fix their parsers and validators
+					)
+				except Exception as _:
+					raise ModelProviderError(message=str(e), status_code=e.response.status_code, model=self.name) from e
+
+		except APIError as e:
+			raise ModelProviderError(message=e.message, model=self.name) from e
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e
+
+	async def _invoke_regular_completion(self, groq_messages) -> ChatInvokeCompletion[str]:
+		"""Handle regular completion without structured output."""
+		chat_completion = await self.get_client().chat.completions.create(
+			messages=groq_messages,
+			model=self.model,
+			service_tier=self.service_tier,
+			temperature=self.temperature,
+			top_p=self.top_p,
+			seed=self.seed,
+		)
+		usage = self._get_usage(chat_completion)
+		return ChatInvokeCompletion(
+			completion=chat_completion.choices[0].message.content or '',
+			usage=usage,
+		)
+
+	async def _invoke_structured_output(self, groq_messages, output_format: type[T]) -> ChatInvokeCompletion[T]:
+		"""Handle structured output using either tool calling or JSON schema."""
+		schema = SchemaOptimizer.create_optimized_json_schema(output_format)
+
+		if self.model in ToolCallingModels:
+			response = await self._invoke_with_tool_calling(groq_messages, output_format, schema)
+		else:
+			response = await self._invoke_with_json_schema(groq_messages, output_format, schema)
+
+		if not response.choices[0].message.content:
+			raise ModelProviderError(
+				message='No content in response',
+				status_code=500,
+				model=self.name,
+			)
+
+		parsed_response = output_format.model_validate_json(response.choices[0].message.content)
+		usage = self._get_usage(response)
+
+		return ChatInvokeCompletion(
+			completion=parsed_response,
+			usage=usage,
+		)
+
+	async def _invoke_with_tool_calling(self, groq_messages, output_format: type[T], schema) -> ChatCompletion:
+		"""Handle structured output using tool calling."""
+		tool = ChatCompletionToolParam(
+			function={
+				'name': output_format.__name__,
+				'description': f'Extract information in the format of {output_format.__name__}',
+				'parameters': schema,
+			},
+			type='function',
+		)
+		tool_choice: ChatCompletionToolChoiceOptionParam = 'required'
+
+		return await self.get_client().chat.completions.create(
+			model=self.model,
+			messages=groq_messages,
+			temperature=self.temperature,
+			top_p=self.top_p,
+			seed=self.seed,
+			tools=[tool],
+			tool_choice=tool_choice,
+			service_tier=self.service_tier,
+		)
+
+	async def _invoke_with_json_schema(self, groq_messages, output_format: type[T], schema) -> ChatCompletion:
+		"""Handle structured output using JSON schema."""
+		return await self.get_client().chat.completions.create(
+			model=self.model,
+			messages=groq_messages,
+			temperature=self.temperature,
+			top_p=self.top_p,
+			seed=self.seed,
+			response_format=ResponseFormatResponseFormatJsonSchema(
+				json_schema=ResponseFormatResponseFormatJsonSchemaJsonSchema(
+					name=output_format.__name__,
+					description='Model output schema',
+					schema=schema,
+				),
+				type='json_schema',
+			),
+			service_tier=self.service_tier,
+		)

chapter9/browser-use-rpa/browser-use/browser_use/llm/groq/parser.py

@@ -0,0 +1,158 @@
+import json
+import logging
+import re
+from typing import TypeVar
+
+from groq import APIStatusError
+from pydantic import BaseModel
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar('T', bound=BaseModel)
+
+
+class ParseFailedGenerationError(Exception):
+	pass
+
+
+def try_parse_groq_failed_generation(
+	error: APIStatusError,
+	output_format: type[T],
+) -> T:
+	"""Extract JSON from model output, handling both plain JSON and code-block-wrapped JSON."""
+	try:
+		content = error.body['error']['failed_generation']  # type: ignore
+
+		# If content is wrapped in code blocks, extract just the JSON part
+		if '```' in content:
+			# Find the JSON content between code blocks
+			content = content.split('```')[1]
+			# Remove language identifier if present (e.g., 'json\n')
+			if '\n' in content:
+				content = content.split('\n', 1)[1]
+
+		# remove html-like tags before the first { and after the last }
+		# This handles cases like <|header_start|>assistant<|header_end|> and <function=AgentOutput>
+		# Only remove content before { if content doesn't already start with {
+		if not content.strip().startswith('{'):
+			content = re.sub(r'^.*?(?=\{)', '', content, flags=re.DOTALL)
+
+		# Remove common HTML-like tags and patterns at the end, but be more conservative
+		# Look for patterns like </function>, <|header_start|>, etc. after the JSON
+		content = re.sub(r'\}(\s*<[^>]*>.*?$)', '}', content, flags=re.DOTALL)
+		content = re.sub(r'\}(\s*<\|[^|]*\|>.*?$)', '}', content, flags=re.DOTALL)
+
+		# Handle extra characters after the JSON, including stray braces
+		# Find the position of the last } that would close the main JSON object
+		content = content.strip()
+
+		if content.endswith('}'):
+			# Try to parse and see if we get valid JSON
+			try:
+				json.loads(content)
+			except json.JSONDecodeError:
+				# If parsing fails, try to find the correct end of the JSON
+				# by counting braces and removing anything after the balanced JSON
+				brace_count = 0
+				last_valid_pos = -1
+				for i, char in enumerate(content):
+					if char == '{':
+						brace_count += 1
+					elif char == '}':
+						brace_count -= 1
+						if brace_count == 0:
+							last_valid_pos = i + 1
+							break
+
+				if last_valid_pos > 0:
+					content = content[:last_valid_pos]
+
+		# Fix control characters in JSON strings before parsing
+		# This handles cases where literal control characters appear in JSON values
+		content = _fix_control_characters_in_json(content)
+
+		# Parse the cleaned content
+		result_dict = json.loads(content)
+
+		# some models occasionally respond with a list containing one dict: https://github.com/browser-use/browser-use/issues/1458
+		if isinstance(result_dict, list) and len(result_dict) == 1 and isinstance(result_dict[0], dict):
+			result_dict = result_dict[0]
+
+		logger.debug(f'Successfully parsed model output: {result_dict}')
+		return output_format.model_validate(result_dict)
+
+	except KeyError as e:
+		raise ParseFailedGenerationError(e) from e
+
+	except json.JSONDecodeError as e:
+		logger.warning(f'Failed to parse model output: {content} {str(e)}')
+		raise ValueError(f'Could not parse response. {str(e)}')
+
+	except Exception as e:
+		raise ParseFailedGenerationError(error.response.text) from e
+
+
+def _fix_control_characters_in_json(content: str) -> str:
+	"""Fix control characters in JSON string values to make them valid JSON."""
+	try:
+		# First try to parse as-is to see if it's already valid
+		json.loads(content)
+		return content
+	except json.JSONDecodeError:
+		pass
+
+	# More sophisticated approach: only escape control characters inside string values
+	# while preserving JSON structure formatting
+
+	result = []
+	i = 0
+	in_string = False
+	escaped = False
+
+	while i < len(content):
+		char = content[i]
+
+		if not in_string:
+			# Outside of string - check if we're entering a string
+			if char == '"':
+				in_string = True
+			result.append(char)
+		else:
+			# Inside string - handle escaping and control characters
+			if escaped:
+				# Previous character was backslash, so this character is escaped
+				result.append(char)
+				escaped = False
+			elif char == '\\':
+				# This is an escape character
+				result.append(char)
+				escaped = True
+			elif char == '"':
+				# End of string
+				result.append(char)
+				in_string = False
+			elif char == '\n':
+				# Literal newline inside string - escape it
+				result.append('\\n')
+			elif char == '\r':
+				# Literal carriage return inside string - escape it
+				result.append('\\r')
+			elif char == '\t':
+				# Literal tab inside string - escape it
+				result.append('\\t')
+			elif char == '\b':
+				# Literal backspace inside string - escape it
+				result.append('\\b')
+			elif char == '\f':
+				# Literal form feed inside string - escape it
+				result.append('\\f')
+			elif ord(char) < 32:
+				# Other control characters inside string - convert to unicode escape
+				result.append(f'\\u{ord(char):04x}')
+			else:
+				# Normal character inside string
+				result.append(char)
+
+		i += 1
+
+	return ''.join(result)

chapter9/browser-use-rpa/browser-use/browser_use/llm/groq/serializer.py

@@ -0,0 +1,159 @@
+from typing import overload
+
+from groq.types.chat import (
+	ChatCompletionAssistantMessageParam,
+	ChatCompletionContentPartImageParam,
+	ChatCompletionContentPartTextParam,
+	ChatCompletionMessageParam,
+	ChatCompletionMessageToolCallParam,
+	ChatCompletionSystemMessageParam,
+	ChatCompletionUserMessageParam,
+)
+from groq.types.chat.chat_completion_content_part_image_param import ImageURL
+from groq.types.chat.chat_completion_message_tool_call_param import Function
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartRefusalParam,
+	ContentPartTextParam,
+	SystemMessage,
+	ToolCall,
+	UserMessage,
+)
+
+
+class GroqMessageSerializer:
+	"""Serializer for converting between custom message types and OpenAI message param types."""
+
+	@staticmethod
+	def _serialize_content_part_text(part: ContentPartTextParam) -> ChatCompletionContentPartTextParam:
+		return ChatCompletionContentPartTextParam(text=part.text, type='text')
+
+	@staticmethod
+	def _serialize_content_part_image(part: ContentPartImageParam) -> ChatCompletionContentPartImageParam:
+		return ChatCompletionContentPartImageParam(
+			image_url=ImageURL(url=part.image_url.url, detail=part.image_url.detail),
+			type='image_url',
+		)
+
+	@staticmethod
+	def _serialize_user_content(
+		content: str | list[ContentPartTextParam | ContentPartImageParam],
+	) -> str | list[ChatCompletionContentPartTextParam | ChatCompletionContentPartImageParam]:
+		"""Serialize content for user messages (text and images allowed)."""
+		if isinstance(content, str):
+			return content
+
+		serialized_parts: list[ChatCompletionContentPartTextParam | ChatCompletionContentPartImageParam] = []
+		for part in content:
+			if part.type == 'text':
+				serialized_parts.append(GroqMessageSerializer._serialize_content_part_text(part))
+			elif part.type == 'image_url':
+				serialized_parts.append(GroqMessageSerializer._serialize_content_part_image(part))
+		return serialized_parts
+
+	@staticmethod
+	def _serialize_system_content(
+		content: str | list[ContentPartTextParam],
+	) -> str:
+		"""Serialize content for system messages (text only)."""
+		if isinstance(content, str):
+			return content
+
+		serialized_parts: list[str] = []
+		for part in content:
+			if part.type == 'text':
+				serialized_parts.append(GroqMessageSerializer._serialize_content_part_text(part)['text'])
+
+		return '\n'.join(serialized_parts)
+
+	@staticmethod
+	def _serialize_assistant_content(
+		content: str | list[ContentPartTextParam | ContentPartRefusalParam] | None,
+	) -> str | None:
+		"""Serialize content for assistant messages (text and refusal allowed)."""
+		if content is None:
+			return None
+		if isinstance(content, str):
+			return content
+
+		serialized_parts: list[str] = []
+		for part in content:
+			if part.type == 'text':
+				serialized_parts.append(GroqMessageSerializer._serialize_content_part_text(part)['text'])
+
+		return '\n'.join(serialized_parts)
+
+	@staticmethod
+	def _serialize_tool_call(tool_call: ToolCall) -> ChatCompletionMessageToolCallParam:
+		return ChatCompletionMessageToolCallParam(
+			id=tool_call.id,
+			function=Function(name=tool_call.function.name, arguments=tool_call.function.arguments),
+			type='function',
+		)
+
+	# endregion
+
+	# region - Serialize overloads
+	@overload
+	@staticmethod
+	def serialize(message: UserMessage) -> ChatCompletionUserMessageParam: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: SystemMessage) -> ChatCompletionSystemMessageParam: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: AssistantMessage) -> ChatCompletionAssistantMessageParam: ...
+
+	@staticmethod
+	def serialize(message: BaseMessage) -> ChatCompletionMessageParam:
+		"""Serialize a custom message to an OpenAI message param."""
+
+		if isinstance(message, UserMessage):
+			user_result: ChatCompletionUserMessageParam = {
+				'role': 'user',
+				'content': GroqMessageSerializer._serialize_user_content(message.content),
+			}
+			if message.name is not None:
+				user_result['name'] = message.name
+			return user_result
+
+		elif isinstance(message, SystemMessage):
+			system_result: ChatCompletionSystemMessageParam = {
+				'role': 'system',
+				'content': GroqMessageSerializer._serialize_system_content(message.content),
+			}
+			if message.name is not None:
+				system_result['name'] = message.name
+			return system_result
+
+		elif isinstance(message, AssistantMessage):
+			# Handle content serialization
+			content = None
+			if message.content is not None:
+				content = GroqMessageSerializer._serialize_assistant_content(message.content)
+
+			assistant_result: ChatCompletionAssistantMessageParam = {'role': 'assistant'}
+
+			# Only add content if it's not None
+			if content is not None:
+				assistant_result['content'] = content
+
+			if message.name is not None:
+				assistant_result['name'] = message.name
+
+			if message.tool_calls:
+				assistant_result['tool_calls'] = [GroqMessageSerializer._serialize_tool_call(tc) for tc in message.tool_calls]
+
+			return assistant_result
+
+		else:
+			raise ValueError(f'Unknown message type: {type(message)}')
+
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> list[ChatCompletionMessageParam]:
+		return [GroqMessageSerializer.serialize(m) for m in messages]

chapter9/browser-use-rpa/browser-use/browser_use/llm/messages.py

@@ -0,0 +1,238 @@
+"""
+This implementation is based on the OpenAI types, while removing all the parts that are not needed for Browser Use.
+"""
+
+# region - Content parts
+from typing import Literal, Union
+
+from openai import BaseModel
+
+
+def _truncate(text: str, max_length: int = 50) -> str:
+	"""Truncate text to max_length characters, adding ellipsis if truncated."""
+	if len(text) <= max_length:
+		return text
+	return text[: max_length - 3] + '...'
+
+
+def _format_image_url(url: str, max_length: int = 50) -> str:
+	"""Format image URL for display, truncating if necessary."""
+	if url.startswith('data:'):
+		# Base64 image
+		media_type = url.split(';')[0].split(':')[1] if ';' in url else 'image'
+		return f'<base64 {media_type}>'
+	else:
+		# Regular URL
+		return _truncate(url, max_length)
+
+
+class ContentPartTextParam(BaseModel):
+	text: str
+	type: Literal['text'] = 'text'
+
+	def __str__(self) -> str:
+		return f'Text: {_truncate(self.text)}'
+
+	def __repr__(self) -> str:
+		return f'ContentPartTextParam(text={_truncate(self.text)})'
+
+
+class ContentPartRefusalParam(BaseModel):
+	refusal: str
+	type: Literal['refusal'] = 'refusal'
+
+	def __str__(self) -> str:
+		return f'Refusal: {_truncate(self.refusal)}'
+
+	def __repr__(self) -> str:
+		return f'ContentPartRefusalParam(refusal={_truncate(repr(self.refusal), 50)})'
+
+
+SupportedImageMediaType = Literal['image/jpeg', 'image/png', 'image/gif', 'image/webp']
+
+
+class ImageURL(BaseModel):
+	url: str
+	"""Either a URL of the image or the base64 encoded image data."""
+	detail: Literal['auto', 'low', 'high'] = 'auto'
+	"""Specifies the detail level of the image.
+
+    Learn more in the
+    [Vision guide](https://platform.openai.com/docs/guides/vision#low-or-high-fidelity-image-understanding).
+    """
+	# needed for Anthropic
+	media_type: SupportedImageMediaType = 'image/png'
+
+	def __str__(self) -> str:
+		url_display = _format_image_url(self.url)
+		return f'🖼️  Image[{self.media_type}, detail={self.detail}]: {url_display}'
+
+	def __repr__(self) -> str:
+		url_repr = _format_image_url(self.url, 30)
+		return f'ImageURL(url={repr(url_repr)}, detail={repr(self.detail)}, media_type={repr(self.media_type)})'
+
+
+class ContentPartImageParam(BaseModel):
+	image_url: ImageURL
+	type: Literal['image_url'] = 'image_url'
+
+	def __str__(self) -> str:
+		return str(self.image_url)
+
+	def __repr__(self) -> str:
+		return f'ContentPartImageParam(image_url={repr(self.image_url)})'
+
+
+class Function(BaseModel):
+	arguments: str
+	"""
+    The arguments to call the function with, as generated by the model in JSON
+    format. Note that the model does not always generate valid JSON, and may
+    hallucinate parameters not defined by your function schema. Validate the
+    arguments in your code before calling your function.
+    """
+	name: str
+	"""The name of the function to call."""
+
+	def __str__(self) -> str:
+		args_preview = _truncate(self.arguments, 80)
+		return f'{self.name}({args_preview})'
+
+	def __repr__(self) -> str:
+		args_repr = _truncate(repr(self.arguments), 50)
+		return f'Function(name={repr(self.name)}, arguments={args_repr})'
+
+
+class ToolCall(BaseModel):
+	id: str
+	"""The ID of the tool call."""
+	function: Function
+	"""The function that the model called."""
+	type: Literal['function'] = 'function'
+	"""The type of the tool. Currently, only `function` is supported."""
+
+	def __str__(self) -> str:
+		return f'ToolCall[{self.id}]: {self.function}'
+
+	def __repr__(self) -> str:
+		return f'ToolCall(id={repr(self.id)}, function={repr(self.function)})'
+
+
+# endregion
+
+
+# region - Message types
+class _MessageBase(BaseModel):
+	"""Base class for all message types"""
+
+	role: Literal['user', 'system', 'assistant']
+
+	cache: bool = False
+	"""Whether to cache this message. This is only applicable when using Anthropic models.
+	"""
+
+
+class UserMessage(_MessageBase):
+	role: Literal['user'] = 'user'
+	"""The role of the messages author, in this case `user`."""
+
+	content: str | list[ContentPartTextParam | ContentPartImageParam]
+	"""The contents of the user message."""
+
+	name: str | None = None
+	"""An optional name for the participant.
+
+    Provides the model information to differentiate between participants of the same
+    role.
+    """
+
+	@property
+	def text(self) -> str:
+		"""
+		Automatically parse the text inside content, whether it's a string or a list of content parts.
+		"""
+		if isinstance(self.content, str):
+			return self.content
+		elif isinstance(self.content, list):
+			return '\n'.join([part.text for part in self.content if part.type == 'text'])
+		else:
+			return ''
+
+	def __str__(self) -> str:
+		return f'UserMessage(content={self.text})'
+
+	def __repr__(self) -> str:
+		return f'UserMessage(content={repr(self.text)})'
+
+
+class SystemMessage(_MessageBase):
+	role: Literal['system'] = 'system'
+	"""The role of the messages author, in this case `system`."""
+
+	content: str | list[ContentPartTextParam]
+	"""The contents of the system message."""
+
+	name: str | None = None
+
+	@property
+	def text(self) -> str:
+		"""
+		Automatically parse the text inside content, whether it's a string or a list of content parts.
+		"""
+		if isinstance(self.content, str):
+			return self.content
+		elif isinstance(self.content, list):
+			return '\n'.join([part.text for part in self.content if part.type == 'text'])
+		else:
+			return ''
+
+	def __str__(self) -> str:
+		return f'SystemMessage(content={self.text})'
+
+	def __repr__(self) -> str:
+		return f'SystemMessage(content={repr(self.text)})'
+
+
+class AssistantMessage(_MessageBase):
+	role: Literal['assistant'] = 'assistant'
+	"""The role of the messages author, in this case `assistant`."""
+
+	content: str | list[ContentPartTextParam | ContentPartRefusalParam] | None
+	"""The contents of the assistant message."""
+
+	name: str | None = None
+
+	refusal: str | None = None
+	"""The refusal message by the assistant."""
+
+	tool_calls: list[ToolCall] = []
+	"""The tool calls generated by the model, such as function calls."""
+
+	@property
+	def text(self) -> str:
+		"""
+		Automatically parse the text inside content, whether it's a string or a list of content parts.
+		"""
+		if isinstance(self.content, str):
+			return self.content
+		elif isinstance(self.content, list):
+			text = ''
+			for part in self.content:
+				if part.type == 'text':
+					text += part.text
+				elif part.type == 'refusal':
+					text += f'[Refusal] {part.refusal}'
+			return text
+		else:
+			return ''
+
+	def __str__(self) -> str:
+		return f'AssistantMessage(content={self.text})'
+
+	def __repr__(self) -> str:
+		return f'AssistantMessage(content={repr(self.text)})'
+
+
+BaseMessage = Union[UserMessage, SystemMessage, AssistantMessage]
+
+# endregion

chapter9/browser-use-rpa/browser-use/browser_use/llm/models.py

@@ -0,0 +1,171 @@
+"""
+Convenient access to LLM models.
+
+Usage:
+    from browser_use import llm
+
+    # Simple model access
+    model = llm.azure_gpt_4_1_mini
+    model = llm.openai_gpt_4o
+    model = llm.google_gemini_2_5_pro
+"""
+
+import os
+from typing import TYPE_CHECKING
+
+from browser_use.llm.azure.chat import ChatAzureOpenAI
+from browser_use.llm.google.chat import ChatGoogle
+from browser_use.llm.openai.chat import ChatOpenAI
+
+if TYPE_CHECKING:
+	from browser_use.llm.base import BaseChatModel
+
+# Type stubs for IDE autocomplete
+openai_gpt_4o: 'BaseChatModel'
+openai_gpt_4o_mini: 'BaseChatModel'
+openai_gpt_4_1_mini: 'BaseChatModel'
+openai_o1: 'BaseChatModel'
+openai_o1_mini: 'BaseChatModel'
+openai_o1_pro: 'BaseChatModel'
+openai_o3: 'BaseChatModel'
+openai_o3_mini: 'BaseChatModel'
+openai_o3_pro: 'BaseChatModel'
+openai_o4_mini: 'BaseChatModel'
+openai_gpt_5: 'BaseChatModel'
+openai_gpt_5_mini: 'BaseChatModel'
+openai_gpt_5_nano: 'BaseChatModel'
+
+azure_gpt_4o: 'BaseChatModel'
+azure_gpt_4o_mini: 'BaseChatModel'
+azure_gpt_4_1_mini: 'BaseChatModel'
+azure_o1: 'BaseChatModel'
+azure_o1_mini: 'BaseChatModel'
+azure_o1_pro: 'BaseChatModel'
+azure_o3: 'BaseChatModel'
+azure_o3_mini: 'BaseChatModel'
+azure_o3_pro: 'BaseChatModel'
+azure_gpt_5: 'BaseChatModel'
+azure_gpt_5_mini: 'BaseChatModel'
+
+google_gemini_2_0_flash: 'BaseChatModel'
+google_gemini_2_0_pro: 'BaseChatModel'
+google_gemini_2_5_pro: 'BaseChatModel'
+google_gemini_2_5_flash: 'BaseChatModel'
+google_gemini_2_5_flash_lite: 'BaseChatModel'
+
+
+def get_llm_by_name(model_name: str):
+	"""
+	Factory function to create LLM instances from string names with API keys from environment.
+
+	Args:
+	    model_name: String name like 'azure_gpt_4_1_mini', 'openai_gpt_4o', etc.
+
+	Returns:
+	    LLM instance with API keys from environment variables
+
+	Raises:
+	    ValueError: If model_name is not recognized
+	"""
+	if not model_name:
+		raise ValueError('Model name cannot be empty')
+
+	# Parse model name
+	parts = model_name.split('_', 1)
+	if len(parts) < 2:
+		raise ValueError(f"Invalid model name format: '{model_name}'. Expected format: 'provider_model_name'")
+
+	provider = parts[0]
+	model_part = parts[1]
+
+	# Convert underscores back to dots/dashes for actual model names
+	if 'gpt_4_1_mini' in model_part:
+		model = model_part.replace('gpt_4_1_mini', 'gpt-4.1-mini')
+	elif 'gpt_4o_mini' in model_part:
+		model = model_part.replace('gpt_4o_mini', 'gpt-4o-mini')
+	elif 'gpt_4o' in model_part:
+		model = model_part.replace('gpt_4o', 'gpt-4o')
+	elif 'gemini_2_0' in model_part:
+		model = model_part.replace('gemini_2_0', 'gemini-2.0').replace('_', '-')
+	elif 'gemini_2_5' in model_part:
+		model = model_part.replace('gemini_2_5', 'gemini-2.5').replace('_', '-')
+	else:
+		model = model_part.replace('_', '-')
+
+	# OpenAI Models
+	if provider == 'openai':
+		api_key = os.getenv('OPENAI_API_KEY')
+		return ChatOpenAI(model=model, api_key=api_key)
+
+	# Azure OpenAI Models
+	elif provider == 'azure':
+		api_key = os.getenv('AZURE_OPENAI_KEY') or os.getenv('AZURE_OPENAI_API_KEY')
+		azure_endpoint = os.getenv('AZURE_OPENAI_ENDPOINT')
+		return ChatAzureOpenAI(model=model, api_key=api_key, azure_endpoint=azure_endpoint)
+
+	# Google Models
+	elif provider == 'google':
+		api_key = os.getenv('GOOGLE_API_KEY')
+		return ChatGoogle(model=model, api_key=api_key)
+
+	else:
+		available_providers = ['openai', 'azure', 'google']
+		raise ValueError(f"Unknown provider: '{provider}'. Available providers: {', '.join(available_providers)}")
+
+
+# Pre-configured model instances (lazy loaded via __getattr__)
+def __getattr__(name: str) -> 'BaseChatModel':
+	"""Create model instances on demand with API keys from environment."""
+	# Handle chat classes first
+	if name == 'ChatOpenAI':
+		return ChatOpenAI  # type: ignore
+	elif name == 'ChatAzureOpenAI':
+		return ChatAzureOpenAI  # type: ignore
+	elif name == 'ChatGoogle':
+		return ChatGoogle  # type: ignore
+
+	# Handle model instances - these are the main use case
+	try:
+		return get_llm_by_name(name)
+	except ValueError:
+		raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+__all__ = [
+	'ChatOpenAI',
+	'ChatAzureOpenAI',
+	'ChatGoogle',
+	'get_llm_by_name',
+	# OpenAI instances - created on demand
+	'openai_gpt_4o',
+	'openai_gpt_4o_mini',
+	'openai_gpt_4_1_mini',
+	'openai_o1',
+	'openai_o1_mini',
+	'openai_o1_pro',
+	'openai_o3',
+	'openai_o3_mini',
+	'openai_o3_pro',
+	'openai_o4_mini',
+	'openai_gpt_5',
+	'openai_gpt_5_mini',
+	'openai_gpt_5_nano',
+	# Azure instances - created on demand
+	'azure_gpt_4o',
+	'azure_gpt_4o_mini',
+	'azure_gpt_4_1_mini',
+	'azure_o1',
+	'azure_o1_mini',
+	'azure_o1_pro',
+	'azure_o3',
+	'azure_o3_mini',
+	'azure_o3_pro',
+	'azure_gpt_5',
+	'azure_gpt_5_mini',
+	# Google instances - created on demand
+	'google_gemini_2_0_flash',
+	'google_gemini_2_0_pro',
+	'google_gemini_2_5_pro',
+	'google_gemini_2_5_flash',
+	'google_gemini_2_5_flash_lite',
+]

chapter9/browser-use-rpa/browser-use/browser_use/llm/ollama/chat.py

@@ -0,0 +1,97 @@
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any, TypeVar, overload
+
+import httpx
+from ollama import AsyncClient as OllamaAsyncClient
+from ollama import Options
+from pydantic import BaseModel
+
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.exceptions import ModelProviderError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.ollama.serializer import OllamaMessageSerializer
+from browser_use.llm.views import ChatInvokeCompletion
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatOllama(BaseChatModel):
+	"""
+	A wrapper around Ollama's chat model.
+	"""
+
+	model: str
+
+	# # Model params
+	# TODO (matic): Why is this commented out?
+	# temperature: float | None = None
+
+	# Client initialization parameters
+	host: str | None = None
+	timeout: float | httpx.Timeout | None = None
+	client_params: dict[str, Any] | None = None
+	ollama_options: Mapping[str, Any] | Options | None = None
+
+	# Static
+	@property
+	def provider(self) -> str:
+		return 'ollama'
+
+	def _get_client_params(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary."""
+		return {
+			'host': self.host,
+			'timeout': self.timeout,
+			'client_params': self.client_params,
+		}
+
+	def get_client(self) -> OllamaAsyncClient:
+		"""
+		Returns an OllamaAsyncClient client.
+		"""
+		return OllamaAsyncClient(host=self.host, timeout=self.timeout, **self.client_params or {})
+
+	@property
+	def name(self) -> str:
+		return self.model
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: None = None) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T]) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		ollama_messages = OllamaMessageSerializer.serialize_messages(messages)
+
+		try:
+			if output_format is None:
+				response = await self.get_client().chat(
+					model=self.model,
+					messages=ollama_messages,
+					options=self.ollama_options,
+				)
+
+				return ChatInvokeCompletion(completion=response.message.content or '', usage=None)
+			else:
+				schema = output_format.model_json_schema()
+
+				response = await self.get_client().chat(
+					model=self.model,
+					messages=ollama_messages,
+					format=schema,
+					options=self.ollama_options,
+				)
+
+				completion = response.message.content or ''
+				if output_format is not None:
+					completion = output_format.model_validate_json(completion)
+
+				return ChatInvokeCompletion(completion=completion, usage=None)
+
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e

chapter9/browser-use-rpa/browser-use/browser_use/llm/ollama/serializer.py

@@ -0,0 +1,143 @@
+import base64
+import json
+from typing import Any, overload
+
+from ollama._types import Image, Message
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	SystemMessage,
+	ToolCall,
+	UserMessage,
+)
+
+
+class OllamaMessageSerializer:
+	"""Serializer for converting between custom message types and Ollama message types."""
+
+	@staticmethod
+	def _extract_text_content(content: Any) -> str:
+		"""Extract text content from message content, ignoring images."""
+		if content is None:
+			return ''
+		if isinstance(content, str):
+			return content
+
+		text_parts: list[str] = []
+		for part in content:
+			if hasattr(part, 'type'):
+				if part.type == 'text':
+					text_parts.append(part.text)
+				elif part.type == 'refusal':
+					text_parts.append(f'[Refusal] {part.refusal}')
+			# Skip image parts as they're handled separately
+
+		return '\n'.join(text_parts)
+
+	@staticmethod
+	def _extract_images(content: Any) -> list[Image]:
+		"""Extract images from message content."""
+		if content is None or isinstance(content, str):
+			return []
+
+		images: list[Image] = []
+		for part in content:
+			if hasattr(part, 'type') and part.type == 'image_url':
+				url = part.image_url.url
+				if url.startswith('data:'):
+					# Handle base64 encoded images
+					# Format: data:image/png;base64,<data>
+					_, data = url.split(',', 1)
+					# Decode base64 to bytes
+					image_bytes = base64.b64decode(data)
+					images.append(Image(value=image_bytes))
+				else:
+					# Handle URL images (Ollama will download them)
+					images.append(Image(value=url))
+
+		return images
+
+	@staticmethod
+	def _serialize_tool_calls(tool_calls: list[ToolCall]) -> list[Message.ToolCall]:
+		"""Convert browser-use ToolCalls to Ollama ToolCalls."""
+		ollama_tool_calls: list[Message.ToolCall] = []
+
+		for tool_call in tool_calls:
+			# Parse arguments from JSON string to dict for Ollama
+			try:
+				arguments_dict = json.loads(tool_call.function.arguments)
+			except json.JSONDecodeError:
+				# If parsing fails, wrap in a dict
+				arguments_dict = {'arguments': tool_call.function.arguments}
+
+			ollama_tool_call = Message.ToolCall(
+				function=Message.ToolCall.Function(name=tool_call.function.name, arguments=arguments_dict)
+			)
+			ollama_tool_calls.append(ollama_tool_call)
+
+		return ollama_tool_calls
+
+	# region - Serialize overloads
+	@overload
+	@staticmethod
+	def serialize(message: UserMessage) -> Message: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: SystemMessage) -> Message: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: AssistantMessage) -> Message: ...
+
+	@staticmethod
+	def serialize(message: BaseMessage) -> Message:
+		"""Serialize a custom message to an Ollama Message."""
+
+		if isinstance(message, UserMessage):
+			text_content = OllamaMessageSerializer._extract_text_content(message.content)
+			images = OllamaMessageSerializer._extract_images(message.content)
+
+			ollama_message = Message(
+				role='user',
+				content=text_content if text_content else None,
+			)
+
+			if images:
+				ollama_message.images = images
+
+			return ollama_message
+
+		elif isinstance(message, SystemMessage):
+			text_content = OllamaMessageSerializer._extract_text_content(message.content)
+
+			return Message(
+				role='system',
+				content=text_content if text_content else None,
+			)
+
+		elif isinstance(message, AssistantMessage):
+			# Handle content
+			text_content = None
+			if message.content is not None:
+				text_content = OllamaMessageSerializer._extract_text_content(message.content)
+
+			ollama_message = Message(
+				role='assistant',
+				content=text_content if text_content else None,
+			)
+
+			# Handle tool calls
+			if message.tool_calls:
+				ollama_message.tool_calls = OllamaMessageSerializer._serialize_tool_calls(message.tool_calls)
+
+			return ollama_message
+
+		else:
+			raise ValueError(f'Unknown message type: {type(message)}')
+
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> list[Message]:
+		"""Serialize a list of browser_use messages to Ollama Messages."""
+		return [OllamaMessageSerializer.serialize(m) for m in messages]

chapter9/browser-use-rpa/browser-use/browser_use/llm/openai/chat.py

@@ -0,0 +1,273 @@
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass, field
+from typing import Any, Literal, TypeVar, overload
+
+import httpx
+from openai import APIConnectionError, APIStatusError, AsyncOpenAI, RateLimitError
+from openai.types.chat import ChatCompletionContentPartTextParam
+from openai.types.chat.chat_completion import ChatCompletion
+from openai.types.shared.chat_model import ChatModel
+from openai.types.shared_params.reasoning_effort import ReasoningEffort
+from openai.types.shared_params.response_format_json_schema import JSONSchema, ResponseFormatJSONSchema
+from pydantic import BaseModel
+
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.exceptions import ModelProviderError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.openai.serializer import OpenAIMessageSerializer
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatOpenAI(BaseChatModel):
+	"""
+	A wrapper around AsyncOpenAI that implements the BaseLLM protocol.
+
+	This class accepts all AsyncOpenAI parameters while adding model
+	and temperature parameters for the LLM interface (if temperature it not `None`).
+	"""
+
+	# Model configuration
+	model: ChatModel | str
+
+	# Model params
+	temperature: float | None = 0.2
+	frequency_penalty: float | None = 0.3  # this avoids infinite generation of \t for models like 4.1-mini
+	reasoning_effort: ReasoningEffort = 'low'
+	seed: int | None = None
+	service_tier: Literal['auto', 'default', 'flex', 'priority', 'scale'] | None = None
+	top_p: float | None = None
+	add_schema_to_system_prompt: bool = False  # Add JSON schema to system prompt instead of using response_format
+
+	# Client initialization parameters
+	api_key: str | None = None
+	organization: str | None = None
+	project: str | None = None
+	base_url: str | httpx.URL | None = None
+	websocket_base_url: str | httpx.URL | None = None
+	timeout: float | httpx.Timeout | None = None
+	max_retries: int = 5  # Increase default retries for automation reliability
+	default_headers: Mapping[str, str] | None = None
+	default_query: Mapping[str, object] | None = None
+	http_client: httpx.AsyncClient | None = None
+	_strict_response_validation: bool = False
+	max_completion_tokens: int | None = 4096
+	reasoning_models: list[ChatModel | str] | None = field(
+		default_factory=lambda: [
+			'o4-mini',
+			'o3',
+			'o3-mini',
+			'o1',
+			'o1-pro',
+			'o3-pro',
+			'gpt-5',
+			'gpt-5-mini',
+			'gpt-5-nano',
+		]
+	)
+
+	# Static
+	@property
+	def provider(self) -> str:
+		return 'openai'
+
+	def _get_client_params(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary."""
+		# Define base client params
+		base_params = {
+			'api_key': self.api_key,
+			'organization': self.organization,
+			'project': self.project,
+			'base_url': self.base_url,
+			'websocket_base_url': self.websocket_base_url,
+			'timeout': self.timeout,
+			'max_retries': self.max_retries,
+			'default_headers': self.default_headers,
+			'default_query': self.default_query,
+			'_strict_response_validation': self._strict_response_validation,
+		}
+
+		# Create client_params dict with non-None values
+		client_params = {k: v for k, v in base_params.items() if v is not None}
+
+		# Add http_client if provided
+		if self.http_client is not None:
+			client_params['http_client'] = self.http_client
+
+		return client_params
+
+	def get_client(self) -> AsyncOpenAI:
+		"""
+		Returns an AsyncOpenAI client.
+
+		Returns:
+			AsyncOpenAI: An instance of the AsyncOpenAI client.
+		"""
+		client_params = self._get_client_params()
+		return AsyncOpenAI(**client_params)
+
+	@property
+	def name(self) -> str:
+		return str(self.model)
+
+	def _get_usage(self, response: ChatCompletion) -> ChatInvokeUsage | None:
+		if response.usage is not None:
+			completion_tokens = response.usage.completion_tokens
+			completion_token_details = response.usage.completion_tokens_details
+			if completion_token_details is not None:
+				reasoning_tokens = completion_token_details.reasoning_tokens
+				if reasoning_tokens is not None:
+					completion_tokens += reasoning_tokens
+
+			usage = ChatInvokeUsage(
+				prompt_tokens=response.usage.prompt_tokens,
+				prompt_cached_tokens=response.usage.prompt_tokens_details.cached_tokens
+				if response.usage.prompt_tokens_details is not None
+				else None,
+				prompt_cache_creation_tokens=None,
+				prompt_image_tokens=None,
+				# Completion
+				completion_tokens=completion_tokens,
+				total_tokens=response.usage.total_tokens,
+			)
+		else:
+			usage = None
+
+		return usage
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: None = None) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T]) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		"""
+		Invoke the model with the given messages.
+
+		Args:
+			messages: List of chat messages
+			output_format: Optional Pydantic model class for structured output
+
+		Returns:
+			Either a string response or an instance of output_format
+		"""
+
+		openai_messages = OpenAIMessageSerializer.serialize_messages(messages)
+
+		try:
+			model_params: dict[str, Any] = {}
+
+			if self.temperature is not None:
+				model_params['temperature'] = self.temperature
+
+			if self.frequency_penalty is not None:
+				model_params['frequency_penalty'] = self.frequency_penalty
+
+			if self.max_completion_tokens is not None:
+				model_params['max_completion_tokens'] = self.max_completion_tokens
+
+			if self.top_p is not None:
+				model_params['top_p'] = self.top_p
+
+			if self.seed is not None:
+				model_params['seed'] = self.seed
+
+			if self.service_tier is not None:
+				model_params['service_tier'] = self.service_tier
+
+			if self.reasoning_models and any(str(m).lower() in str(self.model).lower() for m in self.reasoning_models):
+				model_params['reasoning_effort'] = self.reasoning_effort
+				del model_params['temperature']
+				del model_params['frequency_penalty']
+
+			if output_format is None:
+				# Return string response
+				response = await self.get_client().chat.completions.create(
+					model=self.model,
+					messages=openai_messages,
+					**model_params,
+				)
+
+				usage = self._get_usage(response)
+				return ChatInvokeCompletion(
+					completion=response.choices[0].message.content or '',
+					usage=usage,
+				)
+
+			else:
+				response_format: JSONSchema = {
+					'name': 'agent_output',
+					'strict': True,
+					'schema': SchemaOptimizer.create_optimized_json_schema(output_format),
+				}
+
+				# Add JSON schema to system prompt if requested
+				if self.add_schema_to_system_prompt and openai_messages and openai_messages[0]['role'] == 'system':
+					schema_text = f'\n<json_schema>\n{response_format}\n</json_schema>'
+					if isinstance(openai_messages[0]['content'], str):
+						openai_messages[0]['content'] += schema_text
+					elif isinstance(openai_messages[0]['content'], Iterable):
+						openai_messages[0]['content'] = list(openai_messages[0]['content']) + [
+							ChatCompletionContentPartTextParam(text=schema_text, type='text')
+						]
+
+				# Return structured response
+				response = await self.get_client().chat.completions.create(
+					model=self.model,
+					messages=openai_messages,
+					response_format=ResponseFormatJSONSchema(json_schema=response_format, type='json_schema'),
+					**model_params,
+				)
+
+				if response.choices[0].message.content is None:
+					raise ModelProviderError(
+						message='Failed to parse structured output from model response',
+						status_code=500,
+						model=self.name,
+					)
+
+				usage = self._get_usage(response)
+
+				parsed = output_format.model_validate_json(response.choices[0].message.content)
+
+				return ChatInvokeCompletion(
+					completion=parsed,
+					usage=usage,
+				)
+
+		except RateLimitError as e:
+			error_message = e.response.json().get('error', {})
+			error_message = (
+				error_message.get('message', 'Unknown model error') if isinstance(error_message, dict) else error_message
+			)
+			raise ModelProviderError(
+				message=error_message,
+				status_code=e.response.status_code,
+				model=self.name,
+			) from e
+
+		except APIConnectionError as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e
+
+		except APIStatusError as e:
+			try:
+				error_message = e.response.json().get('error', {})
+			except Exception:
+				error_message = e.response.text
+			error_message = (
+				error_message.get('message', 'Unknown model error') if isinstance(error_message, dict) else error_message
+			)
+			raise ModelProviderError(
+				message=error_message,
+				status_code=e.response.status_code,
+				model=self.name,
+			) from e
+
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e

chapter9/browser-use-rpa/browser-use/browser_use/llm/openai/like.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass
+
+from browser_use.llm.openai.chat import ChatOpenAI
+
+
+@dataclass
+class ChatOpenAILike(ChatOpenAI):
+	"""
+	A class for to interact with any provider using the OpenAI API schema.
+
+	Args:
+	    model (str): The name of the OpenAI model to use.
+	"""
+
+	model: str

chapter9/browser-use-rpa/browser-use/browser_use/llm/openai/serializer.py

@@ -0,0 +1,165 @@
+from typing import overload
+
+from openai.types.chat import (
+	ChatCompletionAssistantMessageParam,
+	ChatCompletionContentPartImageParam,
+	ChatCompletionContentPartRefusalParam,
+	ChatCompletionContentPartTextParam,
+	ChatCompletionMessageFunctionToolCallParam,
+	ChatCompletionMessageParam,
+	ChatCompletionSystemMessageParam,
+	ChatCompletionUserMessageParam,
+)
+from openai.types.chat.chat_completion_content_part_image_param import ImageURL
+from openai.types.chat.chat_completion_message_function_tool_call_param import Function
+
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartRefusalParam,
+	ContentPartTextParam,
+	SystemMessage,
+	ToolCall,
+	UserMessage,
+)
+
+
+class OpenAIMessageSerializer:
+	"""Serializer for converting between custom message types and OpenAI message param types."""
+
+	@staticmethod
+	def _serialize_content_part_text(part: ContentPartTextParam) -> ChatCompletionContentPartTextParam:
+		return ChatCompletionContentPartTextParam(text=part.text, type='text')
+
+	@staticmethod
+	def _serialize_content_part_image(part: ContentPartImageParam) -> ChatCompletionContentPartImageParam:
+		return ChatCompletionContentPartImageParam(
+			image_url=ImageURL(url=part.image_url.url, detail=part.image_url.detail),
+			type='image_url',
+		)
+
+	@staticmethod
+	def _serialize_content_part_refusal(part: ContentPartRefusalParam) -> ChatCompletionContentPartRefusalParam:
+		return ChatCompletionContentPartRefusalParam(refusal=part.refusal, type='refusal')
+
+	@staticmethod
+	def _serialize_user_content(
+		content: str | list[ContentPartTextParam | ContentPartImageParam],
+	) -> str | list[ChatCompletionContentPartTextParam | ChatCompletionContentPartImageParam]:
+		"""Serialize content for user messages (text and images allowed)."""
+		if isinstance(content, str):
+			return content
+
+		serialized_parts: list[ChatCompletionContentPartTextParam | ChatCompletionContentPartImageParam] = []
+		for part in content:
+			if part.type == 'text':
+				serialized_parts.append(OpenAIMessageSerializer._serialize_content_part_text(part))
+			elif part.type == 'image_url':
+				serialized_parts.append(OpenAIMessageSerializer._serialize_content_part_image(part))
+		return serialized_parts
+
+	@staticmethod
+	def _serialize_system_content(
+		content: str | list[ContentPartTextParam],
+	) -> str | list[ChatCompletionContentPartTextParam]:
+		"""Serialize content for system messages (text only)."""
+		if isinstance(content, str):
+			return content
+
+		serialized_parts: list[ChatCompletionContentPartTextParam] = []
+		for part in content:
+			if part.type == 'text':
+				serialized_parts.append(OpenAIMessageSerializer._serialize_content_part_text(part))
+		return serialized_parts
+
+	@staticmethod
+	def _serialize_assistant_content(
+		content: str | list[ContentPartTextParam | ContentPartRefusalParam] | None,
+	) -> str | list[ChatCompletionContentPartTextParam | ChatCompletionContentPartRefusalParam] | None:
+		"""Serialize content for assistant messages (text and refusal allowed)."""
+		if content is None:
+			return None
+		if isinstance(content, str):
+			return content
+
+		serialized_parts: list[ChatCompletionContentPartTextParam | ChatCompletionContentPartRefusalParam] = []
+		for part in content:
+			if part.type == 'text':
+				serialized_parts.append(OpenAIMessageSerializer._serialize_content_part_text(part))
+			elif part.type == 'refusal':
+				serialized_parts.append(OpenAIMessageSerializer._serialize_content_part_refusal(part))
+		return serialized_parts
+
+	@staticmethod
+	def _serialize_tool_call(tool_call: ToolCall) -> ChatCompletionMessageFunctionToolCallParam:
+		return ChatCompletionMessageFunctionToolCallParam(
+			id=tool_call.id,
+			function=Function(name=tool_call.function.name, arguments=tool_call.function.arguments),
+			type='function',
+		)
+
+	# endregion
+
+	# region - Serialize overloads
+	@overload
+	@staticmethod
+	def serialize(message: UserMessage) -> ChatCompletionUserMessageParam: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: SystemMessage) -> ChatCompletionSystemMessageParam: ...
+
+	@overload
+	@staticmethod
+	def serialize(message: AssistantMessage) -> ChatCompletionAssistantMessageParam: ...
+
+	@staticmethod
+	def serialize(message: BaseMessage) -> ChatCompletionMessageParam:
+		"""Serialize a custom message to an OpenAI message param."""
+
+		if isinstance(message, UserMessage):
+			user_result: ChatCompletionUserMessageParam = {
+				'role': 'user',
+				'content': OpenAIMessageSerializer._serialize_user_content(message.content),
+			}
+			if message.name is not None:
+				user_result['name'] = message.name
+			return user_result
+
+		elif isinstance(message, SystemMessage):
+			system_result: ChatCompletionSystemMessageParam = {
+				'role': 'system',
+				'content': OpenAIMessageSerializer._serialize_system_content(message.content),
+			}
+			if message.name is not None:
+				system_result['name'] = message.name
+			return system_result
+
+		elif isinstance(message, AssistantMessage):
+			# Handle content serialization
+			content = None
+			if message.content is not None:
+				content = OpenAIMessageSerializer._serialize_assistant_content(message.content)
+
+			assistant_result: ChatCompletionAssistantMessageParam = {'role': 'assistant'}
+
+			# Only add content if it's not None
+			if content is not None:
+				assistant_result['content'] = content
+
+			if message.name is not None:
+				assistant_result['name'] = message.name
+			if message.refusal is not None:
+				assistant_result['refusal'] = message.refusal
+			if message.tool_calls:
+				assistant_result['tool_calls'] = [OpenAIMessageSerializer._serialize_tool_call(tc) for tc in message.tool_calls]
+
+			return assistant_result
+
+		else:
+			raise ValueError(f'Unknown message type: {type(message)}')
+
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> list[ChatCompletionMessageParam]:
+		return [OpenAIMessageSerializer.serialize(m) for m in messages]

chapter9/browser-use-rpa/browser-use/browser_use/llm/openrouter/chat.py

@@ -0,0 +1,208 @@
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any, TypeVar, overload
+
+import httpx
+from openai import APIConnectionError, APIStatusError, AsyncOpenAI, RateLimitError
+from openai.types.chat.chat_completion import ChatCompletion
+from openai.types.shared_params.response_format_json_schema import (
+	JSONSchema,
+	ResponseFormatJSONSchema,
+)
+from pydantic import BaseModel
+
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.exceptions import ModelProviderError, ModelRateLimitError
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.openrouter.serializer import OpenRouterMessageSerializer
+from browser_use.llm.schema import SchemaOptimizer
+from browser_use.llm.views import ChatInvokeCompletion, ChatInvokeUsage
+
+T = TypeVar('T', bound=BaseModel)
+
+
+@dataclass
+class ChatOpenRouter(BaseChatModel):
+	"""
+	A wrapper around OpenRouter's chat API, which provides access to various LLM models
+	through a unified OpenAI-compatible interface.
+
+	This class implements the BaseChatModel protocol for OpenRouter's API.
+	"""
+
+	# Model configuration
+	model: str
+
+	# Model params
+	temperature: float | None = None
+	top_p: float | None = None
+	seed: int | None = None
+
+	# Client initialization parameters
+	api_key: str | None = None
+	http_referer: str | None = None  # OpenRouter specific parameter for tracking
+	base_url: str | httpx.URL = 'https://openrouter.ai/api/v1'
+	timeout: float | httpx.Timeout | None = None
+	max_retries: int = 10
+	default_headers: Mapping[str, str] | None = None
+	default_query: Mapping[str, object] | None = None
+	http_client: httpx.AsyncClient | None = None
+	_strict_response_validation: bool = False
+
+	# Static
+	@property
+	def provider(self) -> str:
+		return 'openrouter'
+
+	def _get_client_params(self) -> dict[str, Any]:
+		"""Prepare client parameters dictionary."""
+		# Define base client params
+		base_params = {
+			'api_key': self.api_key,
+			'base_url': self.base_url,
+			'timeout': self.timeout,
+			'max_retries': self.max_retries,
+			'default_headers': self.default_headers,
+			'default_query': self.default_query,
+			'_strict_response_validation': self._strict_response_validation,
+			'top_p': self.top_p,
+			'seed': self.seed,
+		}
+
+		# Create client_params dict with non-None values
+		client_params = {k: v for k, v in base_params.items() if v is not None}
+
+		# Add http_client if provided
+		if self.http_client is not None:
+			client_params['http_client'] = self.http_client
+
+		return client_params
+
+	def get_client(self) -> AsyncOpenAI:
+		"""
+		Returns an AsyncOpenAI client configured for OpenRouter.
+
+		Returns:
+		    AsyncOpenAI: An instance of the AsyncOpenAI client with OpenRouter base URL.
+		"""
+		if not hasattr(self, '_client'):
+			client_params = self._get_client_params()
+			self._client = AsyncOpenAI(**client_params)
+		return self._client
+
+	@property
+	def name(self) -> str:
+		return str(self.model)
+
+	def _get_usage(self, response: ChatCompletion) -> ChatInvokeUsage | None:
+		"""Extract usage information from the OpenRouter response."""
+		if response.usage is None:
+			return None
+
+		prompt_details = getattr(response.usage, 'prompt_tokens_details', None)
+		cached_tokens = prompt_details.cached_tokens if prompt_details else None
+
+		return ChatInvokeUsage(
+			prompt_tokens=response.usage.prompt_tokens,
+			prompt_cached_tokens=cached_tokens,
+			prompt_cache_creation_tokens=None,
+			prompt_image_tokens=None,
+			# Completion
+			completion_tokens=response.usage.completion_tokens,
+			total_tokens=response.usage.total_tokens,
+		)
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: None = None) -> ChatInvokeCompletion[str]: ...
+
+	@overload
+	async def ainvoke(self, messages: list[BaseMessage], output_format: type[T]) -> ChatInvokeCompletion[T]: ...
+
+	async def ainvoke(
+		self, messages: list[BaseMessage], output_format: type[T] | None = None
+	) -> ChatInvokeCompletion[T] | ChatInvokeCompletion[str]:
+		"""
+		Invoke the model with the given messages through OpenRouter.
+
+		Args:
+		    messages: List of chat messages
+		    output_format: Optional Pydantic model class for structured output
+
+		Returns:
+		    Either a string response or an instance of output_format
+		"""
+		openrouter_messages = OpenRouterMessageSerializer.serialize_messages(messages)
+
+		# Set up extra headers for OpenRouter
+		extra_headers = {}
+		if self.http_referer:
+			extra_headers['HTTP-Referer'] = self.http_referer
+
+		try:
+			if output_format is None:
+				# Return string response
+				response = await self.get_client().chat.completions.create(
+					model=self.model,
+					messages=openrouter_messages,
+					temperature=self.temperature,
+					top_p=self.top_p,
+					seed=self.seed,
+					extra_headers=extra_headers,
+				)
+
+				usage = self._get_usage(response)
+				return ChatInvokeCompletion(
+					completion=response.choices[0].message.content or '',
+					usage=usage,
+				)
+
+			else:
+				# Create a JSON schema for structured output
+				schema = SchemaOptimizer.create_optimized_json_schema(output_format)
+
+				response_format_schema: JSONSchema = {
+					'name': 'agent_output',
+					'strict': True,
+					'schema': schema,
+				}
+
+				# Return structured response
+				response = await self.get_client().chat.completions.create(
+					model=self.model,
+					messages=openrouter_messages,
+					temperature=self.temperature,
+					top_p=self.top_p,
+					seed=self.seed,
+					response_format=ResponseFormatJSONSchema(
+						json_schema=response_format_schema,
+						type='json_schema',
+					),
+					extra_headers=extra_headers,
+				)
+
+				if response.choices[0].message.content is None:
+					raise ModelProviderError(
+						message='Failed to parse structured output from model response',
+						status_code=500,
+						model=self.name,
+					)
+				usage = self._get_usage(response)
+
+				parsed = output_format.model_validate_json(response.choices[0].message.content)
+
+				return ChatInvokeCompletion(
+					completion=parsed,
+					usage=usage,
+				)
+
+		except RateLimitError as e:
+			raise ModelRateLimitError(message=e.message, model=self.name) from e
+
+		except APIConnectionError as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e
+
+		except APIStatusError as e:
+			raise ModelProviderError(message=e.message, status_code=e.status_code, model=self.name) from e
+
+		except Exception as e:
+			raise ModelProviderError(message=str(e), model=self.name) from e

chapter9/browser-use-rpa/browser-use/browser_use/llm/openrouter/serializer.py

@@ -0,0 +1,26 @@
+from openai.types.chat import ChatCompletionMessageParam
+
+from browser_use.llm.messages import BaseMessage
+from browser_use.llm.openai.serializer import OpenAIMessageSerializer
+
+
+class OpenRouterMessageSerializer:
+	"""
+	Serializer for converting between custom message types and OpenRouter message formats.
+
+	OpenRouter uses the OpenAI-compatible API, so we can reuse the OpenAI serializer.
+	"""
+
+	@staticmethod
+	def serialize_messages(messages: list[BaseMessage]) -> list[ChatCompletionMessageParam]:
+		"""
+		Serialize a list of browser_use messages to OpenRouter-compatible messages.
+
+		Args:
+		    messages: List of browser_use messages
+
+		Returns:
+		    List of OpenRouter-compatible messages (identical to OpenAI format)
+		"""
+		# OpenRouter uses the same message format as OpenAI
+		return OpenAIMessageSerializer.serialize_messages(messages)

chapter9/browser-use-rpa/browser-use/browser_use/llm/schema.py

@@ -0,0 +1,161 @@
+"""
+Utilities for creating optimized Pydantic schemas for LLM usage.
+"""
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class SchemaOptimizer:
+	@staticmethod
+	def create_optimized_json_schema(model: type[BaseModel]) -> dict[str, Any]:
+		"""
+		Create the most optimized schema by flattening all $ref/$defs while preserving
+		FULL descriptions and ALL action definitions. Also ensures OpenAI strict mode compatibility.
+
+		Args:
+			model: The Pydantic model to optimize
+
+		Returns:
+			Optimized schema with all $refs resolved and strict mode compatibility
+		"""
+		# Generate original schema
+		original_schema = model.model_json_schema()
+
+		# Extract $defs for reference resolution, then flatten everything
+		defs_lookup = original_schema.get('$defs', {})
+
+		def optimize_schema(
+			obj: Any,
+			defs_lookup: dict[str, Any] | None = None,
+			*,
+			in_properties: bool = False,  # NEW: track context
+		) -> Any:
+			"""Apply all optimization techniques including flattening all $ref/$defs"""
+			if isinstance(obj, dict):
+				optimized: dict[str, Any] = {}
+				flattened_ref: dict[str, Any] | None = None
+
+				# Skip unnecessary fields AND $defs (we'll inline everything)
+				skip_fields = ['additionalProperties', '$defs']
+
+				for key, value in obj.items():
+					if key in skip_fields:
+						continue
+
+					# Skip metadata "title" unless we're iterating inside an actual `properties` map
+					if key == 'title' and not in_properties:
+						continue
+
+					# Preserve FULL descriptions without truncation
+					elif key == 'description':
+						optimized[key] = value
+
+					# Handle type field
+					elif key == 'type':
+						optimized[key] = value
+
+					# FLATTEN: Resolve $ref by inlining the actual definition
+					elif key == '$ref' and defs_lookup:
+						ref_path = value.split('/')[-1]  # Get the definition name from "#/$defs/SomeName"
+						if ref_path in defs_lookup:
+							# Get the referenced definition and flatten it
+							referenced_def = defs_lookup[ref_path]
+							flattened_ref = optimize_schema(referenced_def, defs_lookup)
+
+					# Keep all anyOf structures (action unions) and resolve any $refs within
+					elif key == 'anyOf' and isinstance(value, list):
+						optimized[key] = [optimize_schema(item, defs_lookup) for item in value]
+
+					# Recursively optimize nested structures
+					elif key in ['properties', 'items']:
+						optimized[key] = optimize_schema(
+							value,
+							defs_lookup,
+							in_properties=(key == 'properties'),
+						)
+
+					# Keep essential validation fields
+					elif key in ['type', 'required', 'minimum', 'maximum', 'minItems', 'maxItems', 'pattern', 'default']:
+						optimized[key] = value if not isinstance(value, (dict, list)) else optimize_schema(value, defs_lookup)
+
+					# Recursively process all other fields
+					else:
+						optimized[key] = optimize_schema(value, defs_lookup) if isinstance(value, (dict, list)) else value
+
+				# If we have a flattened reference, merge it with the optimized properties
+				if flattened_ref is not None and isinstance(flattened_ref, dict):
+					# Start with the flattened reference as the base
+					result = flattened_ref.copy()
+
+					# Merge in any sibling properties that were processed
+					for key, value in optimized.items():
+						# Preserve descriptions from the original object if they exist
+						if key == 'description' and 'description' not in result:
+							result[key] = value
+						elif key != 'description':  # Don't overwrite description from flattened ref
+							result[key] = value
+
+					return result
+				else:
+					# No $ref, just return the optimized object
+					# CRITICAL: Add additionalProperties: false to ALL objects for OpenAI strict mode
+					if optimized.get('type') == 'object':
+						optimized['additionalProperties'] = False
+
+					return optimized
+
+			elif isinstance(obj, list):
+				return [optimize_schema(item, defs_lookup, in_properties=in_properties) for item in obj]
+			return obj
+
+		# Create optimized schema with flattening
+		optimized_result = optimize_schema(original_schema, defs_lookup)
+
+		# Ensure we have a dictionary (should always be the case for schema root)
+		if not isinstance(optimized_result, dict):
+			raise ValueError('Optimized schema result is not a dictionary')
+
+		optimized_schema: dict[str, Any] = optimized_result
+
+		# Additional pass to ensure ALL objects have additionalProperties: false
+		def ensure_additional_properties_false(obj: Any) -> None:
+			"""Ensure all objects have additionalProperties: false"""
+			if isinstance(obj, dict):
+				# If it's an object type, ensure additionalProperties is false
+				if obj.get('type') == 'object':
+					obj['additionalProperties'] = False
+
+				# Recursively apply to all values
+				for value in obj.values():
+					if isinstance(value, (dict, list)):
+						ensure_additional_properties_false(value)
+			elif isinstance(obj, list):
+				for item in obj:
+					if isinstance(item, (dict, list)):
+						ensure_additional_properties_false(item)
+
+		ensure_additional_properties_false(optimized_schema)
+		SchemaOptimizer._make_strict_compatible(optimized_schema)
+
+		return optimized_schema
+
+	@staticmethod
+	def _make_strict_compatible(schema: dict[str, Any] | list[Any]) -> None:
+		"""Ensure all properties are required for OpenAI strict mode"""
+		if isinstance(schema, dict):
+			# First recursively apply to nested objects
+			for key, value in schema.items():
+				if isinstance(value, (dict, list)) and key != 'required':
+					SchemaOptimizer._make_strict_compatible(value)
+
+			# Then update required for this level
+			if 'properties' in schema and 'type' in schema and schema['type'] == 'object':
+				# Add all properties to required array
+				all_props = list(schema['properties'].keys())
+				schema['required'] = all_props  # Set all properties as required
+
+		elif isinstance(schema, list):
+			for item in schema:
+				SchemaOptimizer._make_strict_compatible(item)

chapter9/browser-use-rpa/browser-use/browser_use/llm/tests/test_anthropic_cache.py

@@ -0,0 +1,290 @@
+import logging
+from typing import cast
+
+from browser_use.agent.service import Agent
+from browser_use.llm.anthropic.chat import ChatAnthropic
+from browser_use.llm.anthropic.serializer import AnthropicMessageSerializer, NonSystemMessage
+from browser_use.llm.messages import (
+	AssistantMessage,
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	Function,
+	ImageURL,
+	SystemMessage,
+	ToolCall,
+	UserMessage,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class TestAnthropicCache:
+	"""Comprehensive test for Anthropic cache serialization."""
+
+	def test_cache_basic_functionality(self):
+		"""Test basic cache functionality for all message types."""
+		# Test cache with different message types
+		messages: list[BaseMessage] = [
+			SystemMessage(content='System message!', cache=True),
+			UserMessage(content='User message!', cache=True),
+			AssistantMessage(content='Assistant message!', cache=False),
+		]
+
+		anthropic_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages)
+
+		assert len(anthropic_messages) == 2
+		assert isinstance(system_message, list)
+		assert isinstance(anthropic_messages[0]['content'], list)
+		assert isinstance(anthropic_messages[1]['content'], str)
+
+		# Test cache with assistant message
+		agent_messages: list[BaseMessage] = [
+			SystemMessage(content='System message!'),
+			UserMessage(content='User message!'),
+			AssistantMessage(content='Assistant message!', cache=True),
+		]
+
+		anthropic_messages, system_message = AnthropicMessageSerializer.serialize_messages(agent_messages)
+
+		assert isinstance(system_message, str)
+		assert isinstance(anthropic_messages[0]['content'], str)
+		assert isinstance(anthropic_messages[1]['content'], list)
+
+	def test_cache_with_tool_calls(self):
+		"""Test cache functionality with tool calls."""
+		tool_call = ToolCall(id='test_id', function=Function(name='test_function', arguments='{"arg": "value"}'))
+
+		# Assistant with tool calls and cache
+		assistant_with_tools = AssistantMessage(content='Assistant with tools', tool_calls=[tool_call], cache=True)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([assistant_with_tools])
+
+		assert len(messages) == 1
+		assert isinstance(messages[0]['content'], list)
+		# Should have both text and tool_use blocks
+		assert len(messages[0]['content']) >= 2
+
+	def test_cache_with_images(self):
+		"""Test cache functionality with image content."""
+		user_with_image = UserMessage(
+			content=[
+				ContentPartTextParam(text='Here is an image:', type='text'),
+				ContentPartImageParam(image_url=ImageURL(url='https://example.com/image.jpg'), type='image_url'),
+			],
+			cache=True,
+		)
+
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_with_image])
+
+		assert len(messages) == 1
+		assert isinstance(messages[0]['content'], list)
+		assert len(messages[0]['content']) == 2
+
+	def test_cache_with_base64_images(self):
+		"""Test cache functionality with base64 images."""
+		base64_url = 'data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='
+
+		user_with_base64 = UserMessage(
+			content=[
+				ContentPartTextParam(text='Base64 image:', type='text'),
+				ContentPartImageParam(image_url=ImageURL(url=base64_url), type='image_url'),
+			],
+			cache=True,
+		)
+
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_with_base64])
+
+		assert len(messages) == 1
+		assert isinstance(messages[0]['content'], list)
+
+	def test_cache_content_types(self):
+		"""Test different content types with cache."""
+		# String content with cache should become list
+		user_string_cached = UserMessage(content='String message', cache=True)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_string_cached])
+		assert isinstance(messages[0]['content'], list)
+
+		# String content without cache should remain string
+		user_string_no_cache = UserMessage(content='String message', cache=False)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_string_no_cache])
+		assert isinstance(messages[0]['content'], str)
+
+		# List content maintains list format regardless of cache
+		user_list_cached = UserMessage(content=[ContentPartTextParam(text='List message', type='text')], cache=True)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_list_cached])
+		assert isinstance(messages[0]['content'], list)
+
+		user_list_no_cache = UserMessage(content=[ContentPartTextParam(text='List message', type='text')], cache=False)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([user_list_no_cache])
+		assert isinstance(messages[0]['content'], list)
+
+	def test_assistant_cache_empty_content(self):
+		"""Test AssistantMessage with empty content and cache."""
+		# With cache
+		assistant_empty_cached = AssistantMessage(content=None, cache=True)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([assistant_empty_cached])
+
+		assert len(messages) == 1
+		assert isinstance(messages[0]['content'], list)
+
+		# Without cache
+		assistant_empty_no_cache = AssistantMessage(content=None, cache=False)
+		messages, _ = AnthropicMessageSerializer.serialize_messages([assistant_empty_no_cache])
+
+		assert len(messages) == 1
+		assert isinstance(messages[0]['content'], str)
+
+	def test_mixed_cache_scenarios(self):
+		"""Test various combinations of cached and non-cached messages."""
+		messages_list: list[BaseMessage] = [
+			SystemMessage(content='System with cache', cache=True),
+			UserMessage(content='User with cache', cache=True),
+			AssistantMessage(content='Assistant without cache', cache=False),
+			UserMessage(content='User without cache', cache=False),
+			AssistantMessage(content='Assistant with cache', cache=True),
+		]
+
+		serialized_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages_list)
+
+		# Check system message is cached (becomes list)
+		assert isinstance(system_message, list)
+
+		# Check serialized messages
+		assert len(serialized_messages) == 4
+
+		# User with cache should be list
+		assert isinstance(serialized_messages[0]['content'], list)
+
+		# Assistant without cache should be string
+		assert isinstance(serialized_messages[1]['content'], str)
+
+		# User without cache should be string
+		assert isinstance(serialized_messages[2]['content'], str)
+
+		# Assistant with cache should be list
+		assert isinstance(serialized_messages[3]['content'], list)
+
+	def test_system_message_cache_behavior(self):
+		"""Test SystemMessage specific cache behavior."""
+		# With cache
+		system_cached = SystemMessage(content='System message with cache', cache=True)
+		result = AnthropicMessageSerializer.serialize(system_cached)
+		assert isinstance(result, SystemMessage)
+
+		# Test serialization to string format
+		serialized_content = AnthropicMessageSerializer._serialize_content_to_str(result.content, use_cache=True)
+		assert isinstance(serialized_content, list)
+
+		# Without cache
+		system_no_cache = SystemMessage(content='System message without cache', cache=False)
+		result = AnthropicMessageSerializer.serialize(system_no_cache)
+		assert isinstance(result, SystemMessage)
+
+		serialized_content = AnthropicMessageSerializer._serialize_content_to_str(result.content, use_cache=False)
+		assert isinstance(serialized_content, str)
+
+	def test_agent_messages_integration(self):
+		"""Test integration with actual agent messages."""
+		agent = Agent(task='Hello, world!', llm=ChatAnthropic(''))
+
+		messages = agent.message_manager.get_messages()
+		anthropic_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages)
+
+		# System message should be properly handled
+		assert system_message is not None
+
+	def test_cache_cleaning_last_message_only(self):
+		"""Test that only the last cache=True message remains cached."""
+		# Create multiple messages with cache=True
+		messages_list: list[BaseMessage] = [
+			UserMessage(content='First user message', cache=True),
+			AssistantMessage(content='First assistant message', cache=True),
+			UserMessage(content='Second user message', cache=True),
+			AssistantMessage(content='Second assistant message', cache=False),
+			UserMessage(content='Third user message', cache=True),  # This should be the only one cached
+		]
+
+		# Test the cleaning method directly (only accepts non-system messages)
+		normal_messages = cast(list[NonSystemMessage], [msg for msg in messages_list if not isinstance(msg, SystemMessage)])
+		cleaned_messages = AnthropicMessageSerializer._clean_cache_messages(normal_messages)
+
+		# Verify only the last cache=True message remains cached
+		assert not cleaned_messages[0].cache  # First user message should be uncached
+		assert not cleaned_messages[1].cache  # First assistant message should be uncached
+		assert not cleaned_messages[2].cache  # Second user message should be uncached
+		assert not cleaned_messages[3].cache  # Second assistant message was already uncached
+		assert cleaned_messages[4].cache  # Third user message should remain cached
+
+		# Test through serialize_messages
+		serialized_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages_list)
+
+		# Count how many messages have list content (indicating caching)
+		cached_content_count = sum(1 for msg in serialized_messages if isinstance(msg['content'], list))
+
+		# Only one message should have cached content
+		assert cached_content_count == 1
+
+		# The last message should be the cached one
+		assert isinstance(serialized_messages[-1]['content'], list)
+
+	def test_cache_cleaning_with_system_message(self):
+		"""Test that system messages are not affected by cache cleaning logic."""
+		messages_list: list[BaseMessage] = [
+			SystemMessage(content='System message', cache=True),  # System messages are handled separately
+			UserMessage(content='First user message', cache=True),
+			AssistantMessage(content='Assistant message', cache=True),  # This should be the only normal message cached
+		]
+
+		# Test through serialize_messages to see the full integration
+		serialized_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages_list)
+
+		# System message should be cached
+		assert isinstance(system_message, list)
+
+		# Only one normal message should have cached content (the last one)
+		cached_content_count = sum(1 for msg in serialized_messages if isinstance(msg['content'], list))
+		assert cached_content_count == 1
+
+		# The last message should be the cached one
+		assert isinstance(serialized_messages[-1]['content'], list)
+
+	def test_cache_cleaning_no_cached_messages(self):
+		"""Test that messages without cache=True are not affected."""
+		normal_messages_list = [
+			UserMessage(content='User message 1', cache=False),
+			AssistantMessage(content='Assistant message 1', cache=False),
+			UserMessage(content='User message 2', cache=False),
+		]
+
+		cleaned_messages = AnthropicMessageSerializer._clean_cache_messages(normal_messages_list)
+
+		# All messages should remain uncached
+		for msg in cleaned_messages:
+			assert not msg.cache
+
+	def test_max_4_cache_blocks(self):
+		"""Test that the max number of cache blocks is 4."""
+		agent = Agent(task='Hello, world!', llm=ChatAnthropic(''))
+		messages = agent.message_manager.get_messages()
+		anthropic_messages, system_message = AnthropicMessageSerializer.serialize_messages(messages)
+
+		logger.info(anthropic_messages)
+		logger.info(system_message)
+
+
+if __name__ == '__main__':
+	test_instance = TestAnthropicCache()
+	test_instance.test_cache_basic_functionality()
+	test_instance.test_cache_with_tool_calls()
+	test_instance.test_cache_with_images()
+	test_instance.test_cache_with_base64_images()
+	test_instance.test_cache_content_types()
+	test_instance.test_assistant_cache_empty_content()
+	test_instance.test_mixed_cache_scenarios()
+	test_instance.test_system_message_cache_behavior()
+	test_instance.test_agent_messages_integration()
+	test_instance.test_cache_cleaning_last_message_only()
+	test_instance.test_cache_cleaning_with_system_message()
+	test_instance.test_cache_cleaning_no_cached_messages()
+	test_instance.test_max_4_cache_blocks()
+	print('All cache tests passed!')

chapter9/browser-use-rpa/browser-use/browser_use/llm/tests/test_chat_models.py

@@ -0,0 +1,248 @@
+import os
+
+import pytest
+from pydantic import BaseModel
+
+from browser_use.llm import ChatAnthropic, ChatGoogle, ChatGroq, ChatOpenAI, ChatOpenRouter
+from browser_use.llm.messages import ContentPartTextParam
+
+
+class CapitalResponse(BaseModel):
+	"""Structured response for capital question"""
+
+	country: str
+	capital: str
+
+
+class TestChatModels:
+	from browser_use.llm.messages import (
+		AssistantMessage,
+		BaseMessage,
+		SystemMessage,
+		UserMessage,
+	)
+
+	"""Test suite for all chat model implementations"""
+
+	# Test Constants
+	SYSTEM_MESSAGE = SystemMessage(content=[ContentPartTextParam(text='You are a helpful assistant.', type='text')])
+	FRANCE_QUESTION = UserMessage(content='What is the capital of France? Answer in one word.')
+	FRANCE_ANSWER = AssistantMessage(content='Paris')
+	GERMANY_QUESTION = UserMessage(content='What is the capital of Germany? Answer in one word.')
+
+	# Expected values
+	EXPECTED_GERMANY_CAPITAL = 'berlin'
+	EXPECTED_FRANCE_COUNTRY = 'france'
+	EXPECTED_FRANCE_CAPITAL = 'paris'
+
+	# Test messages for conversation
+	CONVERSATION_MESSAGES: list[BaseMessage] = [
+		SYSTEM_MESSAGE,
+		FRANCE_QUESTION,
+		FRANCE_ANSWER,
+		GERMANY_QUESTION,
+	]
+
+	# Test messages for structured output
+	STRUCTURED_MESSAGES: list[BaseMessage] = [UserMessage(content='What is the capital of France?')]
+
+	# OpenAI Tests
+	@pytest.fixture
+	def openrouter_chat(self):
+		"""Provides an initialized ChatOpenRouter client for tests."""
+		if not os.getenv('OPENROUTER_API_KEY'):
+			pytest.skip('OPENROUTER_API_KEY not set')
+		return ChatOpenRouter(model='openai/gpt-4o-mini', api_key=os.getenv('OPENROUTER_API_KEY'), temperature=0)
+
+	@pytest.mark.asyncio
+	async def test_openai_ainvoke_normal(self):
+		"""Test normal text response from OpenAI"""
+		# Skip if no API key
+		if not os.getenv('OPENAI_API_KEY'):
+			pytest.skip('OPENAI_API_KEY not set')
+
+		chat = ChatOpenAI(model='gpt-4o-mini', temperature=0)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_openai_ainvoke_structured(self):
+		"""Test structured output from OpenAI"""
+		# Skip if no API key
+		if not os.getenv('OPENAI_API_KEY'):
+			pytest.skip('OPENAI_API_KEY not set')
+
+		chat = ChatOpenAI(model='gpt-4o-mini', temperature=0)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL
+
+	# Anthropic Tests
+	@pytest.mark.asyncio
+	async def test_anthropic_ainvoke_normal(self):
+		"""Test normal text response from Anthropic"""
+		# Skip if no API key
+		if not os.getenv('ANTHROPIC_API_KEY'):
+			pytest.skip('ANTHROPIC_API_KEY not set')
+
+		chat = ChatAnthropic(model='claude-3-5-haiku-latest', max_tokens=100, temperature=0)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_anthropic_ainvoke_structured(self):
+		"""Test structured output from Anthropic"""
+		# Skip if no API key
+		if not os.getenv('ANTHROPIC_API_KEY'):
+			pytest.skip('ANTHROPIC_API_KEY not set')
+
+		chat = ChatAnthropic(model='claude-3-5-haiku-latest', max_tokens=100, temperature=0)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL
+
+	# Google Gemini Tests
+	@pytest.mark.asyncio
+	async def test_google_ainvoke_normal(self):
+		"""Test normal text response from Google Gemini"""
+		# Skip if no API key
+		if not os.getenv('GOOGLE_API_KEY'):
+			pytest.skip('GOOGLE_API_KEY not set')
+
+		chat = ChatGoogle(model='gemini-2.0-flash', api_key=os.getenv('GOOGLE_API_KEY'), temperature=0)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_google_ainvoke_structured(self):
+		"""Test structured output from Google Gemini"""
+		# Skip if no API key
+		if not os.getenv('GOOGLE_API_KEY'):
+			pytest.skip('GOOGLE_API_KEY not set')
+
+		chat = ChatGoogle(model='gemini-2.0-flash', api_key=os.getenv('GOOGLE_API_KEY'), temperature=0)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL
+
+	# Google Gemini with Vertex AI Tests
+	@pytest.mark.asyncio
+	async def test_google_vertex_ainvoke_normal(self):
+		"""Test normal text response from Google Gemini via Vertex AI"""
+		# Skip if no project ID
+		if not os.getenv('GOOGLE_CLOUD_PROJECT'):
+			pytest.skip('GOOGLE_CLOUD_PROJECT not set')
+
+		chat = ChatGoogle(
+			model='gemini-2.0-flash',
+			vertexai=True,
+			project=os.getenv('GOOGLE_CLOUD_PROJECT'),
+			location='us-central1',
+			temperature=0,
+		)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_google_vertex_ainvoke_structured(self):
+		"""Test structured output from Google Gemini via Vertex AI"""
+		# Skip if no project ID
+		if not os.getenv('GOOGLE_CLOUD_PROJECT'):
+			pytest.skip('GOOGLE_CLOUD_PROJECT not set')
+
+		chat = ChatGoogle(
+			model='gemini-2.0-flash',
+			vertexai=True,
+			project=os.getenv('GOOGLE_CLOUD_PROJECT'),
+			location='us-central1',
+			temperature=0,
+		)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL
+
+	# Groq Tests
+	@pytest.mark.asyncio
+	async def test_groq_ainvoke_normal(self):
+		"""Test normal text response from Groq"""
+		# Skip if no API key
+		if not os.getenv('GROQ_API_KEY'):
+			pytest.skip('GROQ_API_KEY not set')
+
+		chat = ChatGroq(model='meta-llama/llama-4-maverick-17b-128e-instruct', temperature=0)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_groq_ainvoke_structured(self):
+		"""Test structured output from Groq"""
+		# Skip if no API key
+		if not os.getenv('GROQ_API_KEY'):
+			pytest.skip('GROQ_API_KEY not set')
+
+		chat = ChatGroq(model='meta-llama/llama-4-maverick-17b-128e-instruct', temperature=0)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL
+
+	# OpenRouter Tests
+	@pytest.mark.asyncio
+	async def test_openrouter_ainvoke_normal(self):
+		"""Test normal text response from OpenRouter"""
+		# Skip if no API key
+		if not os.getenv('OPENROUTER_API_KEY'):
+			pytest.skip('OPENROUTER_API_KEY not set')
+
+		chat = ChatOpenRouter(model='openai/gpt-4o-mini', api_key=os.getenv('OPENROUTER_API_KEY'), temperature=0)
+		response = await chat.ainvoke(self.CONVERSATION_MESSAGES)
+		completion = response.completion
+
+		assert isinstance(completion, str)
+		assert self.EXPECTED_GERMANY_CAPITAL in completion.lower()
+
+	@pytest.mark.asyncio
+	async def test_openrouter_ainvoke_structured(self):
+		"""Test structured output from OpenRouter"""
+		# Skip if no API key
+		if not os.getenv('OPENROUTER_API_KEY'):
+			pytest.skip('OPENROUTER_API_KEY not set')
+
+		chat = ChatOpenRouter(model='openai/gpt-4o-mini', api_key=os.getenv('OPENROUTER_API_KEY'), temperature=0)
+		response = await chat.ainvoke(self.STRUCTURED_MESSAGES, output_format=CapitalResponse)
+		completion = response.completion
+
+		assert isinstance(completion, CapitalResponse)
+		assert completion.country.lower() == self.EXPECTED_FRANCE_COUNTRY
+		assert completion.capital.lower() == self.EXPECTED_FRANCE_CAPITAL

chapter9/browser-use-rpa/browser-use/browser_use/llm/tests/test_gemini_image.py

@@ -0,0 +1,91 @@
+import asyncio
+import base64
+import io
+import random
+
+from PIL import Image, ImageDraw, ImageFont
+
+from browser_use.llm.google.chat import ChatGoogle
+from browser_use.llm.google.serializer import GoogleMessageSerializer
+from browser_use.llm.messages import (
+	BaseMessage,
+	ContentPartImageParam,
+	ContentPartTextParam,
+	ImageURL,
+	SystemMessage,
+	UserMessage,
+)
+
+
+def create_random_text_image(text: str = 'hello world', width: int = 4000, height: int = 4000) -> str:
+	# Create image with random background color
+	bg_color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
+	image = Image.new('RGB', (width, height), bg_color)
+	draw = ImageDraw.Draw(image)
+
+	# Try to use a default font, fallback to default if not available
+	try:
+		font = ImageFont.truetype('arial.ttf', 24)
+	except Exception:
+		font = ImageFont.load_default()
+
+	# Calculate text position to center it
+	bbox = draw.textbbox((0, 0), text, font=font)
+	text_width = bbox[2] - bbox[0]
+	text_height = bbox[3] - bbox[1]
+	x = (width - text_width) // 2
+	y = (height - text_height) // 2
+
+	# Draw text with contrasting color
+	text_color = (255 - bg_color[0], 255 - bg_color[1], 255 - bg_color[2])
+	draw.text((x, y), text, fill=text_color, font=font)
+
+	# Convert to base64
+	buffer = io.BytesIO()
+	image.save(buffer, format='PNG')
+	img_data = base64.b64encode(buffer.getvalue()).decode()
+
+	return f'data:image/png;base64,{img_data}'
+
+
+async def test_gemini_image_vision():
+	"""Test Gemini's ability to see and describe images."""
+
+	# Create the LLM
+	llm = ChatGoogle(model='gemini-2.0-flash-exp')
+
+	# Create a random image with text
+	image_data_url = create_random_text_image('Hello Gemini! Can you see this text?')
+
+	# Create messages with image
+	messages: list[BaseMessage] = [
+		SystemMessage(content='You are a helpful assistant that can see and describe images.'),
+		UserMessage(
+			content=[
+				ContentPartTextParam(text='What do you see in this image? Please describe the text and any visual elements.'),
+				ContentPartImageParam(image_url=ImageURL(url=image_data_url)),
+			]
+		),
+	]
+
+	# Serialize messages for Google format
+	serializer = GoogleMessageSerializer()
+	formatted_messages, system_message = serializer.serialize_messages(messages)
+
+	print('Testing Gemini image vision...')
+	print(f'System message: {system_message}')
+
+	# Make the API call
+	try:
+		response = await llm.ainvoke(messages)
+		print('\n=== Gemini Response ===')
+		print(response.completion)
+		print(response.usage)
+		print('=======================')
+	except Exception as e:
+		print(f'Error calling Gemini: {e}')
+		print(f'Error type: {type(e)}')
+
+
+if __name__ == '__main__':
+	asyncio.run(test_gemini_image_vision())

chapter9/browser-use-rpa/browser-use/browser_use/llm/tests/test_groq_loop.py

@@ -0,0 +1,51 @@
+import asyncio
+
+from browser_use.llm import ContentText
+from browser_use.llm.groq.chat import ChatGroq
+from browser_use.llm.messages import SystemMessage, UserMessage
+
+llm = ChatGroq(
+	model='meta-llama/llama-4-maverick-17b-128e-instruct',
+	temperature=0.5,
+)
+# llm = ChatOpenAI(model='gpt-4.1-mini')
+
+
+async def main():
+	from pydantic import BaseModel
+
+	from browser_use.tokens.service import TokenCost
+
+	tk = TokenCost().register_llm(llm)
+
+	class Output(BaseModel):
+		reasoning: str
+		answer: str
+
+	message = [
+		SystemMessage(content='You are a helpful assistant that can answer questions and help with tasks.'),
+		UserMessage(
+			content=[
+				ContentText(
+					text=r"Why is the sky blue? write exactly this into reasoning make sure to output ' with  exactly like in the input : "
+				),
+				ContentText(
+					text="""
+	The user's request is to find the lowest priced women's plus size one piece swimsuit in color black with a customer rating of at least 5 on Kohls.com. I am currently on the homepage of Kohls. The page has a search bar and various category links. To begin, I need to navigate to the women's section and search for swimsuits. I will start by clicking on the 'Women' category link."""
+				),
+			]
+		),
+	]
+
+	for i in range(10):
+		print('-' * 50)
+		print(f'start loop {i}')
+		response = await llm.ainvoke(message, output_format=Output)
+		completion = response.completion
+		print(f'start reasoning: {completion.reasoning}')
+		print(f'answer: {completion.answer}')
+		print('-' * 50)
+
+
+if __name__ == '__main__':
+	asyncio.run(main())

chapter9/browser-use-rpa/browser-use/browser_use/llm/tests/test_single_step.py

@@ -0,0 +1,188 @@
+import logging
+import tempfile
+
+import pytest
+
+from browser_use.agent.prompts import AgentMessagePrompt
+from browser_use.agent.service import Agent
+from browser_use.browser.views import BrowserStateSummary, TabInfo
+from browser_use.dom.views import DOMSelectorMap, EnhancedDOMTreeNode, NodeType, SerializedDOMState, SimplifiedNode
+from browser_use.filesystem.file_system import FileSystem
+from browser_use.llm.anthropic.chat import ChatAnthropic
+from browser_use.llm.azure.chat import ChatAzureOpenAI
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.google.chat import ChatGoogle
+from browser_use.llm.groq.chat import ChatGroq
+from browser_use.llm.openai.chat import ChatOpenAI
+
+# Set logging level to INFO for this module
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def create_mock_state_message(temp_dir: str):
+	"""Create a mock state message with a single clickable element."""
+
+	# Create a mock DOM element with a single clickable button
+	mock_button = EnhancedDOMTreeNode(
+		node_id=1,
+		backend_node_id=1,
+		node_type=NodeType.ELEMENT_NODE,
+		node_name='button',
+		node_value='Click Me',
+		attributes={'id': 'test-button'},
+		is_scrollable=False,
+		is_visible=True,
+		absolute_position=None,
+		session_id=None,
+		target_id='ABCD1234ABCD1234ABCD1234ABCD1234ABCD1234',
+		frame_id=None,
+		content_document=None,
+		shadow_root_type=None,
+		shadow_roots=None,
+		parent_node=None,
+		children_nodes=None,
+		ax_node=None,
+		snapshot_node=None,
+	)
+
+	# Create selector map
+	selector_map: DOMSelectorMap = {1: mock_button}
+
+	# Create mock tab info with proper target_id
+	mock_tab = TabInfo(
+		target_id='ABCD1234ABCD1234ABCD1234ABCD1234ABCD1234',
+		url='https://example.com',
+		title='Test Page',
+	)
+
+	dom_state = SerializedDOMState(
+		_root=SimplifiedNode(
+			original_node=mock_button,
+			children=[],
+			should_display=True,
+			interactive_index=1,
+		),
+		selector_map=selector_map,
+	)
+
+	# Create mock browser state with required selector_map
+	mock_browser_state = BrowserStateSummary(
+		dom_state=dom_state,  # Using the actual DOM element
+		url='https://example.com',
+		title='Test Page',
+		tabs=[mock_tab],
+		screenshot='',  # Empty screenshot
+		pixels_above=0,
+		pixels_below=0,
+	)
+
+	# Create file system using the provided temp directory
+	mock_file_system = FileSystem(temp_dir)
+
+	# Create the agent message prompt
+	agent_prompt = AgentMessagePrompt(
+		browser_state_summary=mock_browser_state,
+		file_system=mock_file_system,  # Now using actual FileSystem instance
+		agent_history_description='',  # Empty history
+		read_state_description='',  # Empty read state
+		task='Click the button on the page',
+		include_attributes=['id'],
+		step_info=None,
+		page_filtered_actions=None,
+		max_clickable_elements_length=40000,
+		sensitive_data=None,
+	)
+
+	# Override the clickable_elements_to_string method to return our simple element
+	dom_state.llm_representation = lambda include_attributes=None: '[1]<button id="test-button">Click Me</button>'
+
+	# Get the formatted message
+	message = agent_prompt.get_user_message(use_vision=False)
+
+	return message
+
+
+# Pytest parameterized version
+@pytest.mark.parametrize(
+	'llm_class,model_name',
+	[
+		(ChatGroq, 'meta-llama/llama-4-maverick-17b-128e-instruct'),
+		(ChatGoogle, 'gemini-2.0-flash-exp'),
+		(ChatOpenAI, 'gpt-4.1-mini'),
+		(ChatAnthropic, 'claude-3-5-sonnet-latest'),
+		(ChatAzureOpenAI, 'gpt-4.1-mini'),
+	],
+)
+async def test_single_step_parametrized(llm_class, model_name):
+	"""Test single step with different LLM providers using pytest parametrize."""
+	llm = llm_class(model=model_name)
+
+	agent = Agent(task='Click the button on the page', llm=llm)
+
+	# Create temporary directory that will stay alive during the test
+	with tempfile.TemporaryDirectory() as temp_dir:
+		# Create mock state message
+		mock_message = create_mock_state_message(temp_dir)
+
+		agent.message_manager._set_message_with_type(mock_message, 'state')
+
+		messages = agent.message_manager.get_messages()
+
+		# Test with simple question
+		response = await llm.ainvoke(messages, agent.AgentOutput)
+
+		# Basic assertions to ensure response is valid
+		assert response.completion is not None
+		assert response.usage is not None
+		assert response.usage.total_tokens > 0
+
+
+async def test_single_step():
+	"""Original test function that tests all models in a loop."""
+	# Create a list of models to test
+	models: list[BaseChatModel] = [
+		ChatGroq(model='meta-llama/llama-4-maverick-17b-128e-instruct'),
+		ChatGoogle(model='gemini-2.0-flash-exp'),
+		ChatOpenAI(model='gpt-4.1'),
+		ChatAnthropic(model='claude-3-5-sonnet-latest'),  # Using haiku for cost efficiency
+		ChatAzureOpenAI(model='gpt-4o-mini'),
+	]
+
+	for llm in models:
+		print(f'\n{"=" * 60}')
+		print(f'Testing with model: {llm.provider} - {llm.model}')
+		print(f'{"=" * 60}\n')
+
+		agent = Agent(task='Click the button on the page', llm=llm)
+
+		# Create temporary directory that will stay alive during the test
+		with tempfile.TemporaryDirectory() as temp_dir:
+			# Create mock state message
+			mock_message = create_mock_state_message(temp_dir)
+
+			# Print the mock message content to see what it looks like
+			print('Mock state message:')
+			print(mock_message.content)
+			print('\n' + '=' * 50 + '\n')
+
+			agent.message_manager._set_message_with_type(mock_message, 'state')
+
+			messages = agent.message_manager.get_messages()
+
+			# Test with simple question
+			try:
+				response = await llm.ainvoke(messages, agent.AgentOutput)
+				logger.info(f'Response from {llm.provider}: {response.completion}')
+				logger.info(f'Actions: {str(response.completion.action)}')
+
+			except Exception as e:
+				logger.error(f'Error with {llm.provider}: {type(e).__name__}: {str(e)}')
+
+		print(f'\n{"=" * 60}\n')
+
+
+if __name__ == '__main__':
+	import asyncio
+
+	asyncio.run(test_single_step())

chapter9/browser-use-rpa/browser-use/browser_use/llm/views.py

@@ -0,0 +1,45 @@
+from typing import Generic, TypeVar, Union
+
+from pydantic import BaseModel
+
+T = TypeVar('T', bound=Union[BaseModel, str])
+
+
+class ChatInvokeUsage(BaseModel):
+	"""
+	Usage information for a chat model invocation.
+	"""
+
+	prompt_tokens: int
+	"""The number of tokens in the prompt (this includes the cached tokens as well. When calculating the cost, subtract the cached tokens from the prompt tokens)"""
+
+	prompt_cached_tokens: int | None
+	"""The number of cached tokens."""
+
+	prompt_cache_creation_tokens: int | None
+	"""Anthropic only: The number of tokens used to create the cache."""
+
+	prompt_image_tokens: int | None
+	"""Google only: The number of tokens in the image (prompt tokens is the text tokens + image tokens in that case)"""
+
+	completion_tokens: int
+	"""The number of tokens in the completion."""
+
+	total_tokens: int
+	"""The total number of tokens in the response."""
+
+
+class ChatInvokeCompletion(BaseModel, Generic[T]):
+	"""
+	Response from a chat model invocation.
+	"""
+
+	completion: T
+	"""The completion of the response."""
+
+	# Thinking stuff
+	thinking: str | None = None
+	redacted_thinking: str | None = None
+
+	usage: ChatInvokeUsage | None
+	"""The usage of the response."""