"""DOM watchdog for browser DOM tree management using CDP.""" import asyncio import time from typing import TYPE_CHECKING from browser_use.browser.events import ( BrowserErrorEvent, BrowserStateRequestEvent, ScreenshotEvent, TabCreatedEvent, ) from browser_use.browser.watchdog_base import BaseWatchdog from browser_use.dom.service import DomService from browser_use.dom.views import ( EnhancedDOMTreeNode, SerializedDOMState, ) from browser_use.observability import observe_debug from browser_use.utils import time_execution_async if TYPE_CHECKING: from browser_use.browser.views import BrowserStateSummary, PageInfo class DOMWatchdog(BaseWatchdog): """Handles DOM tree building, serialization, and element access via CDP. This watchdog acts as a bridge between the event-driven browser session and the DomService implementation, maintaining cached state and providing helper methods for other watchdogs. """ LISTENS_TO = [TabCreatedEvent, BrowserStateRequestEvent] EMITS = [BrowserErrorEvent] # Public properties for other watchdogs selector_map: dict[int, EnhancedDOMTreeNode] | None = None current_dom_state: SerializedDOMState | None = None enhanced_dom_tree: EnhancedDOMTreeNode | None = None # Internal DOM service _dom_service: DomService | None = None async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None: # self.logger.debug('Setting up init scripts in browser') return None def _get_recent_events_str(self, limit: int = 10) -> str | None: """Get the most recent events from the event bus as JSON. Args: limit: Maximum number of recent events to include Returns: JSON string of recent events or None if not available """ import json try: # Get all events from history, sorted by creation time (most recent first) all_events = sorted( self.browser_session.event_bus.event_history.values(), key=lambda e: e.event_created_at.timestamp(), reverse=True ) # Take the most recent events and create JSON-serializable data recent_events_data = [] for event in all_events[:limit]: event_data = { 'event_type': event.event_type, 'timestamp': event.event_created_at.isoformat(), } # Add specific fields for certain event types if hasattr(event, 'url'): event_data['url'] = getattr(event, 'url') if hasattr(event, 'error_message'): event_data['error_message'] = getattr(event, 'error_message') if hasattr(event, 'target_id'): event_data['target_id'] = getattr(event, 'target_id') recent_events_data.append(event_data) return json.dumps(recent_events_data) # Return empty array if no events except Exception as e: self.logger.debug(f'Failed to get recent events: {e}') return json.dumps([]) # Return empty JSON array on error @observe_debug(ignore_input=True, ignore_output=True, name='browser_state_request_event') async def on_BrowserStateRequestEvent(self, event: BrowserStateRequestEvent) -> 'BrowserStateSummary': """Handle browser state request by coordinating DOM building and screenshot capture. This is the main entry point for getting the complete browser state. Args: event: The browser state request event with options Returns: Complete BrowserStateSummary with DOM, screenshot, and target info """ from browser_use.browser.views import BrowserStateSummary, PageInfo self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: STARTING browser state request') page_url = await self.browser_session.get_current_page_url() self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got page URL: {page_url}') if self.browser_session.agent_focus: self.logger.debug( f'Current page URL: {page_url}, target_id: {self.browser_session.agent_focus.target_id}, session_id: {self.browser_session.agent_focus.session_id}' ) else: self.logger.debug(f'Current page URL: {page_url}, no cdp_session attached') # check if we should skip DOM tree build for pointless pages not_a_meaningful_website = page_url.lower().split(':', 1)[0] not in ('http', 'https') # Wait for page stability using browser profile settings (main branch pattern) if not not_a_meaningful_website: self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ⏳ Waiting for page stability...') try: await self._wait_for_stable_network() self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Page stability complete') except Exception as e: self.logger.warning( f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Network waiting failed: {e}, continuing anyway...' ) # Get tabs info once at the beginning for all paths self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting tabs info...') tabs_info = await self.browser_session.get_tabs() self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got {len(tabs_info)} tabs') self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Tabs info: {tabs_info}') # Get viewport / scroll position info, remember changing scroll position should invalidate selector_map cache because it only includes visible elements # cdp_session = await self.browser_session.get_or_create_cdp_session(focus=True) # scroll_info = await cdp_session.cdp_client.send.Runtime.evaluate( # params={'expression': 'JSON.stringify({y: document.body.scrollTop, x: document.body.scrollLeft, width: document.documentElement.clientWidth, height: document.documentElement.clientHeight})'}, # session_id=cdp_session.session_id, # ) # self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got scroll info: {scroll_info["result"]}') try: # Fast path for empty pages if not_a_meaningful_website: self.logger.debug(f'⚡ Skipping BuildDOMTree for empty target: {page_url}') self.logger.debug(f'📸 Not taking screenshot for empty page: {page_url} (non-http/https URL)') # Create minimal DOM state content = SerializedDOMState(_root=None, selector_map={}) # Skip screenshot for empty pages screenshot_b64 = None # Try to get page info from CDP, fall back to defaults if unavailable try: page_info = await self._get_page_info() except Exception as e: self.logger.debug(f'Failed to get page info from CDP for empty page: {e}, using fallback') # Use default viewport dimensions viewport = self.browser_session.browser_profile.viewport or {'width': 1280, 'height': 720} page_info = PageInfo( viewport_width=viewport['width'], viewport_height=viewport['height'], page_width=viewport['width'], page_height=viewport['height'], scroll_x=0, scroll_y=0, pixels_above=0, pixels_below=0, pixels_left=0, pixels_right=0, ) return BrowserStateSummary( dom_state=content, url=page_url, title='Empty Tab', tabs=tabs_info, screenshot=screenshot_b64, page_info=page_info, pixels_above=0, pixels_below=0, browser_errors=[], is_pdf_viewer=False, recent_events=self._get_recent_events_str() if event.include_recent_events else None, ) # Execute DOM building and screenshot capture in parallel dom_task = None screenshot_task = None # Start DOM building task if requested if event.include_dom: self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🌳 Starting DOM tree build task...') previous_state = ( self.browser_session._cached_browser_state_summary.dom_state if self.browser_session._cached_browser_state_summary else None ) dom_task = asyncio.create_task(self._build_dom_tree_without_highlights(previous_state)) # Start clean screenshot task if requested (without JS highlights) if event.include_screenshot: self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Starting clean screenshot task...') screenshot_task = asyncio.create_task(self._capture_clean_screenshot()) # Wait for both tasks to complete content = None screenshot_b64 = None if dom_task: try: content = await dom_task self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ DOM tree build completed') except Exception as e: self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: DOM build failed: {e}, using minimal state') content = SerializedDOMState(_root=None, selector_map={}) else: content = SerializedDOMState(_root=None, selector_map={}) if screenshot_task: try: screenshot_b64 = await screenshot_task self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Clean screenshot captured') except Exception as e: self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Clean screenshot failed: {e}') screenshot_b64 = None # Apply Python-based highlighting if both DOM and screenshot are available if screenshot_b64 and content and content.selector_map and self.browser_session.browser_profile.highlight_elements: try: self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🎨 Applying Python-based highlighting...') from browser_use.browser.python_highlights import create_highlighted_screenshot_async # Get CDP session for viewport info cdp_session = await self.browser_session.get_or_create_cdp_session() start = time.time() screenshot_b64 = await create_highlighted_screenshot_async( screenshot_b64, content.selector_map, cdp_session, self.browser_session.browser_profile.filter_highlight_ids, ) self.logger.debug( f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Applied highlights to {len(content.selector_map)} elements in {time.time() - start:.2f}s' ) except Exception as e: self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Python highlighting failed: {e}') # Ensure we have valid content if not content: content = SerializedDOMState(_root=None, selector_map={}) # Tabs info already fetched at the beginning # Get target title safely try: self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting page title...') title = await asyncio.wait_for(self.browser_session.get_current_page_title(), timeout=1.0) self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got title: {title}') except Exception as e: self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Failed to get title: {e}') title = 'Page' # Get comprehensive page info from CDP with timeout try: self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting page info from CDP...') page_info = await asyncio.wait_for(self._get_page_info(), timeout=1.0) self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got page info from CDP: {page_info}') except Exception as e: self.logger.debug( f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Failed to get page info from CDP: {e}, using fallback' ) # Fallback to default viewport dimensions viewport = self.browser_session.browser_profile.viewport or {'width': 1280, 'height': 720} page_info = PageInfo( viewport_width=viewport['width'], viewport_height=viewport['height'], page_width=viewport['width'], page_height=viewport['height'], scroll_x=0, scroll_y=0, pixels_above=0, pixels_below=0, pixels_left=0, pixels_right=0, ) # Check for PDF viewer is_pdf_viewer = page_url.endswith('.pdf') or '/pdf/' in page_url # Build and cache the browser state summary if screenshot_b64: self.logger.debug( f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Creating BrowserStateSummary with screenshot, length: {len(screenshot_b64)}' ) else: self.logger.debug( '🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Creating BrowserStateSummary WITHOUT screenshot' ) browser_state = BrowserStateSummary( dom_state=content, url=page_url, title=title, tabs=tabs_info, screenshot=screenshot_b64, page_info=page_info, pixels_above=0, pixels_below=0, browser_errors=[], is_pdf_viewer=is_pdf_viewer, recent_events=self._get_recent_events_str() if event.include_recent_events else None, ) # Cache the state self.browser_session._cached_browser_state_summary = browser_state self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ COMPLETED - Returning browser state') return browser_state except Exception as e: self.logger.error(f'Failed to get browser state: {e}') # Return minimal recovery state return BrowserStateSummary( dom_state=SerializedDOMState(_root=None, selector_map={}), url=page_url if 'page_url' in locals() else '', title='Error', tabs=[], screenshot=None, page_info=PageInfo( viewport_width=1280, viewport_height=720, page_width=1280, page_height=720, scroll_x=0, scroll_y=0, pixels_above=0, pixels_below=0, pixels_left=0, pixels_right=0, ), pixels_above=0, pixels_below=0, browser_errors=[str(e)], is_pdf_viewer=False, recent_events=None, ) @time_execution_async('build_dom_tree_without_highlights') @observe_debug(ignore_input=True, ignore_output=True, name='build_dom_tree_without_highlights') async def _build_dom_tree_without_highlights(self, previous_state: SerializedDOMState | None = None) -> SerializedDOMState: """Build DOM tree without injecting JavaScript highlights (for parallel execution).""" try: self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: STARTING DOM tree build') # Create or reuse DOM service if self._dom_service is None: self._dom_service = DomService( browser_session=self.browser_session, logger=self.logger, cross_origin_iframes=self.browser_session.browser_profile.cross_origin_iframes, paint_order_filtering=self.browser_session.browser_profile.paint_order_filtering, max_iframes=self.browser_session.browser_profile.max_iframes, max_iframe_depth=self.browser_session.browser_profile.max_iframe_depth, ) # Get serialized DOM tree using the service self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Calling DomService.get_serialized_dom_tree...') start = time.time() self.current_dom_state, self.enhanced_dom_tree, timing_info = await self._dom_service.get_serialized_dom_tree( previous_cached_state=previous_state, ) end = time.time() self.logger.debug( '🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ DomService.get_serialized_dom_tree completed' ) self.logger.debug(f'Time taken to get DOM tree: {end - start} seconds') self.logger.debug(f'Timing breakdown: {timing_info}') # Update selector map for other watchdogs self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Updating selector maps...') self.selector_map = self.current_dom_state.selector_map # Update BrowserSession's cached selector map if self.browser_session: self.browser_session.update_cached_selector_map(self.selector_map) self.logger.debug( f'🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ Selector maps updated, {len(self.selector_map)} elements' ) # Skip JavaScript highlighting injection - Python highlighting will be applied later self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ COMPLETED DOM tree build (no JS highlights)') return self.current_dom_state except Exception as e: self.logger.error(f'Failed to build DOM tree without highlights: {e}') self.event_bus.dispatch( BrowserErrorEvent( error_type='DOMBuildFailed', message=str(e), ) ) raise @time_execution_async('capture_clean_screenshot') @observe_debug(ignore_input=True, ignore_output=True, name='capture_clean_screenshot') async def _capture_clean_screenshot(self) -> str: """Capture a clean screenshot without JavaScript highlights.""" try: self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: Capturing clean screenshot...') # Ensure we have a focused CDP session assert self.browser_session.agent_focus is not None, 'No current target ID' await self.browser_session.get_or_create_cdp_session(target_id=self.browser_session.agent_focus.target_id, focus=True) # Check if handler is registered handlers = self.event_bus.handlers.get('ScreenshotEvent', []) handler_names = [getattr(h, '__name__', str(h)) for h in handlers] self.logger.debug(f'📸 ScreenshotEvent handlers registered: {len(handlers)} - {handler_names}') screenshot_event = self.event_bus.dispatch(ScreenshotEvent(full_page=False)) self.logger.debug('📸 Dispatched ScreenshotEvent, waiting for event to complete...') # Wait for the event itself to complete (this waits for all handlers) await screenshot_event # Get the single handler result screenshot_b64 = await screenshot_event.event_result(raise_if_any=True, raise_if_none=True) if screenshot_b64 is None: raise RuntimeError('Screenshot handler returned None') self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: ✅ Clean screenshot captured successfully') return str(screenshot_b64) except TimeoutError: self.logger.warning('📸 Clean screenshot timed out after 6 seconds - no handler registered or slow page?') raise except Exception as e: self.logger.warning(f'📸 Clean screenshot failed: {type(e).__name__}: {e}') raise async def _wait_for_stable_network(self): """Wait for page stability - simplified for CDP-only branch.""" start_time = time.time() # Apply minimum wait time first (let page settle) min_wait = self.browser_session.browser_profile.minimum_wait_page_load_time if min_wait > 0: self.logger.debug(f'⏳ Minimum wait: {min_wait}s') await asyncio.sleep(min_wait) # Apply network idle wait time (for dynamic content like iframes) network_idle_wait = self.browser_session.browser_profile.wait_for_network_idle_page_load_time if network_idle_wait > 0: self.logger.debug(f'⏳ Network idle wait: {network_idle_wait}s') await asyncio.sleep(network_idle_wait) elapsed = time.time() - start_time self.logger.debug(f'✅ Page stability wait completed in {elapsed:.2f}s') async def _get_page_info(self) -> 'PageInfo': """Get comprehensive page information using a single CDP call. TODO: should we make this an event as well? Returns: PageInfo with all viewport, page dimensions, and scroll information """ from browser_use.browser.views import PageInfo # Get CDP session for the current target if not self.browser_session.agent_focus: raise RuntimeError('No active CDP session - browser may not be connected yet') cdp_session = await self.browser_session.get_or_create_cdp_session( target_id=self.browser_session.agent_focus.target_id, focus=True ) # Get layout metrics which includes all the information we need metrics = await asyncio.wait_for( cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id), timeout=10.0 ) # Extract different viewport types layout_viewport = metrics.get('layoutViewport', {}) visual_viewport = metrics.get('visualViewport', {}) css_visual_viewport = metrics.get('cssVisualViewport', {}) css_layout_viewport = metrics.get('cssLayoutViewport', {}) content_size = metrics.get('contentSize', {}) # Calculate device pixel ratio to convert between device pixels and CSS pixels # This matches the approach in dom/service.py _get_viewport_ratio method css_width = css_visual_viewport.get('clientWidth', css_layout_viewport.get('clientWidth', 1280.0)) device_width = visual_viewport.get('clientWidth', css_width) device_pixel_ratio = device_width / css_width if css_width > 0 else 1.0 # For viewport dimensions, use CSS pixels (what JavaScript sees) # Prioritize CSS layout viewport, then fall back to layout viewport viewport_width = int(css_layout_viewport.get('clientWidth') or layout_viewport.get('clientWidth', 1280)) viewport_height = int(css_layout_viewport.get('clientHeight') or layout_viewport.get('clientHeight', 720)) # For total page dimensions, content size is typically in device pixels, so convert to CSS pixels # by dividing by device pixel ratio raw_page_width = content_size.get('width', viewport_width * device_pixel_ratio) raw_page_height = content_size.get('height', viewport_height * device_pixel_ratio) page_width = int(raw_page_width / device_pixel_ratio) page_height = int(raw_page_height / device_pixel_ratio) # For scroll position, use CSS visual viewport if available, otherwise CSS layout viewport # These should already be in CSS pixels scroll_x = int(css_visual_viewport.get('pageX') or css_layout_viewport.get('pageX', 0)) scroll_y = int(css_visual_viewport.get('pageY') or css_layout_viewport.get('pageY', 0)) # Calculate scroll information - pixels that are above/below/left/right of current viewport pixels_above = scroll_y pixels_below = max(0, page_height - viewport_height - scroll_y) pixels_left = scroll_x pixels_right = max(0, page_width - viewport_width - scroll_x) page_info = PageInfo( viewport_width=viewport_width, viewport_height=viewport_height, page_width=page_width, page_height=page_height, scroll_x=scroll_x, scroll_y=scroll_y, pixels_above=pixels_above, pixels_below=pixels_below, pixels_left=pixels_left, pixels_right=pixels_right, ) return page_info # ========== Public Helper Methods ========== async def get_element_by_index(self, index: int) -> EnhancedDOMTreeNode | None: """Get DOM element by index from cached selector map. Builds DOM if not cached. Returns: EnhancedDOMTreeNode or None if index not found """ if not self.selector_map: # Build DOM if not cached await self._build_dom_tree_without_highlights() return self.selector_map.get(index) if self.selector_map else None def clear_cache(self) -> None: """Clear cached DOM state to force rebuild on next access.""" self.selector_map = None self.current_dom_state = None self.enhanced_dom_tree = None # Keep the DOM service instance to reuse its CDP client connection def is_file_input(self, element: EnhancedDOMTreeNode) -> bool: """Check if element is a file input.""" return element.node_name.upper() == 'INPUT' and element.attributes.get('type', '').lower() == 'file' @staticmethod def is_element_visible_according_to_all_parents(node: EnhancedDOMTreeNode, html_frames: list[EnhancedDOMTreeNode]) -> bool: """Check if the element is visible according to all its parent HTML frames. Delegates to the DomService static method. """ return DomService.is_element_visible_according_to_all_parents(node, html_frames) async def __aexit__(self, exc_type, exc_value, traceback): """Clean up DOM service on exit.""" if self._dom_service: await self._dom_service.__aexit__(exc_type, exc_value, traceback) self._dom_service = None def __del__(self): """Clean up DOM service on deletion.""" super().__del__() # DOM service will clean up its own CDP client self._dom_service = None