"""Downloads watchdog for monitoring and handling file downloads.""" import asyncio import json import os import tempfile from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar from urllib.parse import urlparse import anyio from bubus import BaseEvent from cdp_use.cdp.browser import DownloadProgressEvent, DownloadWillBeginEvent from cdp_use.cdp.target import SessionID, TargetID from pydantic import PrivateAttr from browser_use.browser.events import ( BrowserLaunchEvent, BrowserStateRequestEvent, BrowserStoppedEvent, FileDownloadedEvent, NavigationCompleteEvent, TabClosedEvent, TabCreatedEvent, ) from browser_use.browser.watchdog_base import BaseWatchdog if TYPE_CHECKING: pass class DownloadsWatchdog(BaseWatchdog): """Monitors downloads and handles file download events.""" # Events this watchdog listens to (for documentation) LISTENS_TO: ClassVar[list[type[BaseEvent[Any]]]] = [ BrowserLaunchEvent, BrowserStateRequestEvent, BrowserStoppedEvent, TabCreatedEvent, TabClosedEvent, NavigationCompleteEvent, ] # Events this watchdog emits EMITS: ClassVar[list[type[BaseEvent[Any]]]] = [ FileDownloadedEvent, ] # Private state _sessions_with_listeners: set[str] = PrivateAttr(default_factory=set) # Track sessions that already have download listeners _active_downloads: dict[str, Any] = PrivateAttr(default_factory=dict) _pdf_viewer_cache: dict[str, bool] = PrivateAttr(default_factory=dict) # Cache PDF viewer status by target URL _download_cdp_session_setup: bool = PrivateAttr(default=False) # Track if CDP session is set up _download_cdp_session: Any = PrivateAttr(default=None) # Store CDP session reference _cdp_event_tasks: set[asyncio.Task] = PrivateAttr(default_factory=set) # Track CDP event handler tasks _cdp_downloads_info: dict[str, dict[str, Any]] = PrivateAttr(default_factory=dict) # Map guid -> info _use_js_fetch_for_local: bool = PrivateAttr(default=False) # Guard JS fetch path for local regular downloads async def on_BrowserLaunchEvent(self, event: BrowserLaunchEvent) -> None: self.logger.debug(f'[DownloadsWatchdog] Received BrowserLaunchEvent, EventBus ID: {id(self.event_bus)}') # Ensure downloads directory exists downloads_path = self.browser_session.browser_profile.downloads_path if downloads_path: expanded_path = Path(downloads_path).expanduser().resolve() expanded_path.mkdir(parents=True, exist_ok=True) self.logger.debug(f'[DownloadsWatchdog] Ensured downloads directory exists: {expanded_path}') async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None: """Monitor new tabs for downloads.""" # logger.info(f'[DownloadsWatchdog] TabCreatedEvent received for tab {event.target_id[-4:]}: {event.url}') # Assert downloads path is configured (should always be set by BrowserProfile default) assert self.browser_session.browser_profile.downloads_path is not None, 'Downloads path must be configured' if event.target_id: # logger.info(f'[DownloadsWatchdog] Found target for tab {event.target_id}, calling attach_to_target') await self.attach_to_target(event.target_id) else: self.logger.warning(f'[DownloadsWatchdog] No target found for tab {event.target_id}') async def on_TabClosedEvent(self, event: TabClosedEvent) -> None: """Stop monitoring closed tabs.""" pass # No cleanup needed, browser context handles target lifecycle async def on_BrowserStateRequestEvent(self, event: BrowserStateRequestEvent) -> None: """Handle browser state request events.""" cdp_session = self.browser_session.agent_focus if not cdp_session: return url = await self.browser_session.get_current_page_url() if not url: return target_id = cdp_session.target_id self.event_bus.dispatch( NavigationCompleteEvent( event_type='NavigationCompleteEvent', url=url, target_id=target_id, event_parent_id=event.event_id, ) ) async def on_BrowserStoppedEvent(self, event: BrowserStoppedEvent) -> None: """Clean up when browser stops.""" # Cancel all CDP event handler tasks for task in list(self._cdp_event_tasks): if not task.done(): task.cancel() # Wait for all tasks to complete cancellation if self._cdp_event_tasks: await asyncio.gather(*self._cdp_event_tasks, return_exceptions=True) self._cdp_event_tasks.clear() # Clean up CDP session # CDP sessions are now cached and managed by BrowserSession self._download_cdp_session = None self._download_cdp_session_setup = False # Clear other state self._sessions_with_listeners.clear() self._active_downloads.clear() self._pdf_viewer_cache.clear() async def on_NavigationCompleteEvent(self, event: NavigationCompleteEvent) -> None: """Check for PDFs after navigation completes.""" self.logger.debug(f'[DownloadsWatchdog] NavigationCompleteEvent received for {event.url}, tab #{event.target_id[-4:]}') # Clear PDF cache for the navigated URL since content may have changed if event.url in self._pdf_viewer_cache: del self._pdf_viewer_cache[event.url] # Check if auto-download is enabled auto_download_enabled = self._is_auto_download_enabled() if not auto_download_enabled: return # Note: Using network-based PDF detection that doesn't require JavaScript target_id = event.target_id self.logger.debug(f'[DownloadsWatchdog] Got target_id={target_id} for tab #{event.target_id[-4:]}') is_pdf = await self.check_for_pdf_viewer(target_id) if is_pdf: self.logger.debug(f'[DownloadsWatchdog] 📄 PDF detected at {event.url}, triggering auto-download...') download_path = await self.trigger_pdf_download(target_id) if not download_path: self.logger.warning(f'[DownloadsWatchdog] ⚠️ PDF download failed for {event.url}') def _is_auto_download_enabled(self) -> bool: """Check if auto-download PDFs is enabled in browser profile.""" return self.browser_session.browser_profile.auto_download_pdfs async def attach_to_target(self, target_id: TargetID) -> None: """Set up download monitoring for a specific target.""" # Define CDP event handlers outside of try to avoid indentation/scope issues async def download_will_begin_handler(event: DownloadWillBeginEvent, session_id: SessionID | None): self.logger.debug(f'[DownloadsWatchdog] Download will begin: {event}') # Cache info for later completion event handling (esp. remote browsers) guid = event.get('guid', '') try: suggested_filename = event.get('suggestedFilename') assert suggested_filename, 'CDP DownloadWillBegin missing suggestedFilename' self._cdp_downloads_info[guid] = { 'url': event.get('url', ''), 'suggested_filename': suggested_filename, 'handled': False, } except (AssertionError, KeyError): pass # Create and track the task task = asyncio.create_task(self._handle_cdp_download(event, target_id, session_id)) self._cdp_event_tasks.add(task) # Remove from set when done task.add_done_callback(lambda t: self._cdp_event_tasks.discard(t)) async def download_progress_handler(event: DownloadProgressEvent, session_id: SessionID | None): # Check if download is complete if event.get('state') == 'completed': file_path = event.get('filePath') guid = event.get('guid', '') if self.browser_session.is_local: if file_path: self.logger.debug(f'[DownloadsWatchdog] Download completed: {file_path}') # Track the download self._track_download(file_path) # Mark as handled to prevent fallback duplicate dispatch try: if guid in self._cdp_downloads_info: self._cdp_downloads_info[guid]['handled'] = True except (KeyError, AttributeError): pass else: # No local file path provided, local polling in _handle_cdp_download will handle it self.logger.debug( '[DownloadsWatchdog] No filePath in progress event (local); polling will handle detection' ) else: # Remote browser: do not touch local filesystem. Fallback to downloadPath+suggestedFilename info = self._cdp_downloads_info.get(guid, {}) try: suggested_filename = info.get('suggested_filename') or (Path(file_path).name if file_path else 'download') downloads_path = str(self.browser_session.browser_profile.downloads_path or '') effective_path = file_path or str(Path(downloads_path) / suggested_filename) file_name = Path(effective_path).name file_ext = Path(file_name).suffix.lower().lstrip('.') self.event_bus.dispatch( FileDownloadedEvent( url=info.get('url', ''), path=str(effective_path), file_name=file_name, file_size=0, file_type=file_ext if file_ext else None, ) ) self.logger.debug(f'[DownloadsWatchdog] ✅ (remote) Download completed: {effective_path}') finally: if guid in self._cdp_downloads_info: del self._cdp_downloads_info[guid] try: downloads_path_raw = self.browser_session.browser_profile.downloads_path if not downloads_path_raw: # logger.info(f'[DownloadsWatchdog] No downloads path configured, skipping target: {target_id}') return # No downloads path configured # Check if we already have a download listener on this session # to prevent duplicate listeners from being added # Note: Since download listeners are set up once per browser session, not per target, # we just track if we've set up the browser-level listener if self._download_cdp_session_setup: self.logger.debug('[DownloadsWatchdog] Download listener already set up for browser session') return # logger.debug(f'[DownloadsWatchdog] Setting up CDP download listener for target: {target_id}') # Use CDP session for download events but store reference in watchdog if not self._download_cdp_session_setup: # Set up CDP session for downloads (only once per browser session) cdp_client = self.browser_session.cdp_client # Set download behavior to allow downloads and enable events downloads_path = self.browser_session.browser_profile.downloads_path if not downloads_path: self.logger.warning('[DownloadsWatchdog] No downloads path configured, skipping CDP download setup') return # Ensure path is properly expanded (~ -> absolute path) expanded_downloads_path = Path(downloads_path).expanduser().resolve() await cdp_client.send.Browser.setDownloadBehavior( params={ 'behavior': 'allow', 'downloadPath': str(expanded_downloads_path), # Use expanded absolute path 'eventsEnabled': True, } ) # Register the handlers with CDP cdp_client.register.Browser.downloadWillBegin(download_will_begin_handler) # type: ignore[arg-type] cdp_client.register.Browser.downloadProgress(download_progress_handler) # type: ignore[arg-type] self._download_cdp_session_setup = True self.logger.debug('[DownloadsWatchdog] Set up CDP download listeners') # No need to track individual targets since download listener is browser-level # logger.debug(f'[DownloadsWatchdog] Successfully set up CDP download listener for target: {target_id}') except Exception as e: self.logger.warning(f'[DownloadsWatchdog] Failed to set up CDP download listener for target {target_id}: {e}') def _track_download(self, file_path: str) -> None: """Track a completed download and dispatch the appropriate event. Args: file_path: The path to the downloaded file """ try: # Get file info path = Path(file_path) if path.exists(): file_size = path.stat().st_size self.logger.debug(f'[DownloadsWatchdog] Tracked download: {path.name} ({file_size} bytes)') # Dispatch download event from browser_use.browser.events import FileDownloadedEvent self.event_bus.dispatch( FileDownloadedEvent( url=str(path), # Use the file path as URL for local files path=str(path), file_name=path.name, file_size=file_size, ) ) else: self.logger.warning(f'[DownloadsWatchdog] Downloaded file not found: {file_path}') except Exception as e: self.logger.error(f'[DownloadsWatchdog] Error tracking download: {e}') async def _handle_cdp_download( self, event: DownloadWillBeginEvent, target_id: TargetID, session_id: SessionID | None ) -> None: """Handle a CDP Page.downloadWillBegin event.""" downloads_dir = ( Path( self.browser_session.browser_profile.downloads_path or f'{tempfile.gettempdir()}/browser_use_downloads.{str(self.browser_session.id)[-4:]}' ) .expanduser() .resolve() ) # Ensure path is properly expanded # Initialize variables that may be used outside try blocks unique_filename = None file_size = 0 expected_path = None download_result = None download_url = event.get('url', '') suggested_filename = event.get('suggestedFilename', 'download') guid = event.get('guid', '') try: self.logger.debug(f'[DownloadsWatchdog] ⬇️ File download starting: {suggested_filename} from {download_url[:100]}...') self.logger.debug(f'[DownloadsWatchdog] Full CDP event: {event}') # Since Browser.setDownloadBehavior is already configured, the browser will download the file # We just need to wait for it to appear in the downloads directory expected_path = downloads_dir / suggested_filename # Debug: List current directory contents self.logger.debug(f'[DownloadsWatchdog] Downloads directory: {downloads_dir}') if downloads_dir.exists(): files_before = list(downloads_dir.iterdir()) self.logger.debug(f'[DownloadsWatchdog] Files before download: {[f.name for f in files_before]}') # Try manual JavaScript fetch as a fallback for local browsers (disabled for regular local downloads) if self.browser_session.is_local and self._use_js_fetch_for_local: self.logger.debug(f'[DownloadsWatchdog] Attempting JS fetch fallback for {download_url}') unique_filename = None file_size = None download_result = None try: # Escape the URL for JavaScript import json escaped_url = json.dumps(download_url) # Get the proper session for the frame that initiated the download cdp_session = await self.browser_session.cdp_client_for_frame(event.get('frameId')) assert cdp_session result = await cdp_session.cdp_client.send.Runtime.evaluate( params={ 'expression': f""" (async () => {{ try {{ const response = await fetch({escaped_url}); if (!response.ok) {{ throw new Error(`HTTP error! status: ${{response.status}}`); }} const blob = await response.blob(); const arrayBuffer = await blob.arrayBuffer(); const uint8Array = new Uint8Array(arrayBuffer); return {{ data: Array.from(uint8Array), size: uint8Array.length, contentType: response.headers.get('content-type') || 'application/octet-stream' }}; }} catch (error) {{ throw new Error(`Fetch failed: ${{error.message}}`); }} }})() """, 'awaitPromise': True, 'returnByValue': True, }, session_id=cdp_session.session_id, ) download_result = result.get('result', {}).get('value') if download_result and download_result.get('data'): # Save the file file_data = bytes(download_result['data']) file_size = len(file_data) # Ensure unique filename unique_filename = await self._get_unique_filename(str(downloads_dir), suggested_filename) final_path = downloads_dir / unique_filename # Write the file import anyio async with await anyio.open_file(final_path, 'wb') as f: await f.write(file_data) self.logger.debug(f'[DownloadsWatchdog] ✅ Downloaded and saved file: {final_path} ({file_size} bytes)') expected_path = final_path # Emit download event immediately file_ext = expected_path.suffix.lower().lstrip('.') file_type = file_ext if file_ext else None self.event_bus.dispatch( FileDownloadedEvent( url=download_url, path=str(expected_path), file_name=unique_filename or expected_path.name, file_size=file_size or 0, file_type=file_type, mime_type=(download_result.get('contentType') if download_result else None), from_cache=False, auto_download=False, ) ) # Mark as handled to prevent duplicate dispatch from progress/polling paths try: if guid in self._cdp_downloads_info: self._cdp_downloads_info[guid]['handled'] = True except (KeyError, AttributeError): pass self.logger.debug( f'[DownloadsWatchdog] ✅ File download completed via CDP: {suggested_filename} ({file_size} bytes) saved to {expected_path}' ) return else: self.logger.error('[DownloadsWatchdog] ❌ No data received from fetch') except Exception as fetch_error: self.logger.error(f'[DownloadsWatchdog] ❌ Failed to download file via fetch: {fetch_error}') # For remote browsers, don't poll local filesystem; downloadProgress handler will emit the event if not self.browser_session.is_local: return except Exception as e: self.logger.error(f'[DownloadsWatchdog] ❌ Error handling CDP download: {type(e).__name__} {e}') # If we reach here, the fetch method failed, so wait for native download # Poll the downloads directory for new files self.logger.debug(f'[DownloadsWatchdog] Checking if browser auto-download saved the file for us: {suggested_filename}') # Get initial list of files in downloads directory initial_files = set() if Path(downloads_dir).exists(): for f in Path(downloads_dir).iterdir(): if f.is_file() and not f.name.startswith('.'): initial_files.add(f.name) # Poll for new files max_wait = 20 # seconds start_time = asyncio.get_event_loop().time() while asyncio.get_event_loop().time() - start_time < max_wait: await asyncio.sleep(5.0) # Check every 5 seconds if Path(downloads_dir).exists(): for file_path in Path(downloads_dir).iterdir(): # Skip hidden files and files that were already there if file_path.is_file() and not file_path.name.startswith('.') and file_path.name not in initial_files: # Check if file has content (> 4 bytes) try: file_size = file_path.stat().st_size if file_size > 4: # Found a new download! self.logger.debug( f'[DownloadsWatchdog] ✅ Found downloaded file: {file_path} ({file_size} bytes)' ) # Determine file type from extension file_ext = file_path.suffix.lower().lstrip('.') file_type = file_ext if file_ext else None # Dispatch download event # Skip if already handled by progress/JS fetch info = self._cdp_downloads_info.get(guid, {}) if info.get('handled'): return self.event_bus.dispatch( FileDownloadedEvent( url=download_url, path=str(file_path), file_name=file_path.name, file_size=file_size, file_type=file_type, ) ) # Mark as handled after dispatch try: if guid in self._cdp_downloads_info: self._cdp_downloads_info[guid]['handled'] = True except (KeyError, AttributeError): pass return except Exception as e: self.logger.debug(f'[DownloadsWatchdog] Error checking file {file_path}: {e}') self.logger.warning(f'[DownloadsWatchdog] Download did not complete within {max_wait} seconds') async def _handle_download(self, download: Any) -> None: """Handle a download event.""" download_id = f'{id(download)}' self._active_downloads[download_id] = download self.logger.debug(f'[DownloadsWatchdog] ⬇️ Handling download: {download.suggested_filename} from {download.url[:100]}...') # Debug: Check if download is already being handled elsewhere failure = ( await download.failure() ) # TODO: it always fails for some reason, figure out why connect_over_cdp makes accept_downloads not work self.logger.warning(f'[DownloadsWatchdog] ❌ Download state - canceled: {failure}, url: {download.url}') # logger.info(f'[DownloadsWatchdog] Active downloads count: {len(self._active_downloads)}') try: current_step = 'getting_download_info' # Get download info immediately url = download.url suggested_filename = download.suggested_filename current_step = 'determining_download_directory' # Determine download directory from browser profile downloads_dir = self.browser_session.browser_profile.downloads_path if not downloads_dir: downloads_dir = str(Path.home() / 'Downloads') else: downloads_dir = str(downloads_dir) # Ensure it's a string # Check if Playwright already auto-downloaded the file (due to CDP setup) original_path = Path(downloads_dir) / suggested_filename if original_path.exists() and original_path.stat().st_size > 0: self.logger.debug( f'[DownloadsWatchdog] File already downloaded by Playwright: {original_path} ({original_path.stat().st_size} bytes)' ) # Use the existing file instead of creating a duplicate download_path = original_path file_size = original_path.stat().st_size unique_filename = suggested_filename else: current_step = 'generating_unique_filename' # Ensure unique filename unique_filename = await self._get_unique_filename(downloads_dir, suggested_filename) download_path = Path(downloads_dir) / unique_filename self.logger.debug(f'[DownloadsWatchdog] Download started: {unique_filename} from {url[:100]}...') current_step = 'calling_save_as' # Save the download using Playwright's save_as method self.logger.debug(f'[DownloadsWatchdog] Saving download to: {download_path}') self.logger.debug(f'[DownloadsWatchdog] Download path exists: {download_path.parent.exists()}') self.logger.debug(f'[DownloadsWatchdog] Download path writable: {os.access(download_path.parent, os.W_OK)}') try: self.logger.debug('[DownloadsWatchdog] About to call download.save_as()...') await download.save_as(str(download_path)) self.logger.debug(f'[DownloadsWatchdog] Successfully saved download to: {download_path}') current_step = 'save_as_completed' except Exception as save_error: self.logger.error(f'[DownloadsWatchdog] save_as() failed with error: {save_error}') raise save_error # Get file info file_size = download_path.stat().st_size if download_path.exists() else 0 # Determine file type from extension file_ext = download_path.suffix.lower().lstrip('.') file_type = file_ext if file_ext else None # Try to get MIME type from response headers if available mime_type = None # Note: Playwright doesn't expose response headers directly from Download object # Check if this was a PDF auto-download auto_download = False if file_type == 'pdf': auto_download = self._is_auto_download_enabled() # Emit download event self.event_bus.dispatch( FileDownloadedEvent( url=url, path=str(download_path), file_name=suggested_filename, file_size=file_size, file_type=file_type, mime_type=mime_type, from_cache=False, auto_download=auto_download, ) ) self.logger.debug( f'[DownloadsWatchdog] ✅ Download completed: {suggested_filename} ({file_size} bytes) saved to {download_path}' ) # File is now tracked on filesystem, no need to track in memory except Exception as e: self.logger.error( f'[DownloadsWatchdog] Error handling download at step "{locals().get("current_step", "unknown")}", error: {e}' ) self.logger.error( f'[DownloadsWatchdog] Download state - URL: {download.url}, filename: {download.suggested_filename}' ) finally: # Clean up tracking if download_id in self._active_downloads: del self._active_downloads[download_id] async def check_for_pdf_viewer(self, target_id: TargetID) -> bool: """Check if the current target is a PDF using network-based detection. This method avoids JavaScript execution that can crash WebSocket connections. Returns True if a PDF is detected and should be downloaded. """ self.logger.debug(f'[DownloadsWatchdog] Checking if target {target_id} is PDF viewer...') # Get target info to get URL cdp_client = self.browser_session.cdp_client targets = await cdp_client.send.Target.getTargets() target_info = next((t for t in targets['targetInfos'] if t['targetId'] == target_id), None) if not target_info: self.logger.warning(f'[DownloadsWatchdog] No target info found for {target_id}') return False page_url = target_info.get('url', '') # Check cache first if page_url in self._pdf_viewer_cache: cached_result = self._pdf_viewer_cache[page_url] self.logger.debug(f'[DownloadsWatchdog] Using cached PDF check result for {page_url}: {cached_result}') return cached_result try: # Method 1: Check URL patterns (fastest, most reliable) url_is_pdf = self._check_url_for_pdf(page_url) if url_is_pdf: self.logger.debug(f'[DownloadsWatchdog] PDF detected via URL pattern: {page_url}') self._pdf_viewer_cache[page_url] = True return True # Method 2: Check network response headers via CDP (safer than JavaScript) header_is_pdf = await self._check_network_headers_for_pdf(target_id) if header_is_pdf: self.logger.debug(f'[DownloadsWatchdog] PDF detected via network headers: {page_url}') self._pdf_viewer_cache[page_url] = True return True # Method 3: Check Chrome's PDF viewer specific URLs chrome_pdf_viewer = self._is_chrome_pdf_viewer_url(page_url) if chrome_pdf_viewer: self.logger.debug(f'[DownloadsWatchdog] Chrome PDF viewer detected: {page_url}') self._pdf_viewer_cache[page_url] = True return True # Not a PDF self._pdf_viewer_cache[page_url] = False return False except Exception as e: self.logger.warning(f'[DownloadsWatchdog] ❌ Error checking for PDF viewer: {e}') self._pdf_viewer_cache[page_url] = False return False def _check_url_for_pdf(self, url: str) -> bool: """Check if URL indicates a PDF file.""" if not url: return False url_lower = url.lower() # Direct PDF file extensions if url_lower.endswith('.pdf'): return True # PDF in path if '.pdf' in url_lower: return True # PDF MIME type in URL parameters if any( param in url_lower for param in [ 'content-type=application/pdf', 'content-type=application%2fpdf', 'mimetype=application/pdf', 'type=application/pdf', ] ): return True return False def _is_chrome_pdf_viewer_url(self, url: str) -> bool: """Check if this is Chrome's internal PDF viewer URL.""" if not url: return False url_lower = url.lower() # Chrome PDF viewer uses chrome-extension:// URLs if 'chrome-extension://' in url_lower and 'pdf' in url_lower: return True # Chrome PDF viewer internal URLs if url_lower.startswith('chrome://') and 'pdf' in url_lower: return True return False async def _check_network_headers_for_pdf(self, target_id: TargetID) -> bool: """Infer PDF via navigation history/URL; headers are not available post-navigation in this context.""" try: import asyncio # Get CDP session temp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=False) # Get navigation history to find the main resource history = await asyncio.wait_for( temp_session.cdp_client.send.Page.getNavigationHistory(session_id=temp_session.session_id), timeout=3.0 ) current_entry = history.get('entries', []) if current_entry: current_index = history.get('currentIndex', 0) if 0 <= current_index < len(current_entry): current_url = current_entry[current_index].get('url', '') # Check if the URL itself suggests PDF if self._check_url_for_pdf(current_url): return True # Note: CDP doesn't easily expose response headers for completed navigations # For more complex cases, we'd need to set up Network.responseReceived listeners # before navigation, but that's overkill for most PDF detection cases return False except Exception as e: self.logger.debug(f'[DownloadsWatchdog] Network headers check failed (non-critical): {e}') return False async def trigger_pdf_download(self, target_id: TargetID) -> str | None: """Trigger download of a PDF from Chrome's PDF viewer. Returns the download path if successful, None otherwise. """ self.logger.debug(f'[DownloadsWatchdog] trigger_pdf_download called for target_id={target_id}') if not self.browser_session.browser_profile.downloads_path: self.logger.warning('[DownloadsWatchdog] ❌ No downloads path configured, cannot save PDF download') return None downloads_path = self.browser_session.browser_profile.downloads_path self.logger.debug(f'[DownloadsWatchdog] Downloads path: {downloads_path}') try: # Create a temporary CDP session for this target without switching focus import asyncio self.logger.debug(f'[DownloadsWatchdog] Creating CDP session for PDF download from target {target_id}') temp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=False) # Try to get the PDF URL with timeout result = await asyncio.wait_for( temp_session.cdp_client.send.Runtime.evaluate( params={ 'expression': """ (() => { // For Chrome's PDF viewer, the actual URL is in window.location.href // The embed element's src is often "about:blank" const embedElement = document.querySelector('embed[type="application/x-google-chrome-pdf"]') || document.querySelector('embed[type="application/pdf"]'); if (embedElement) { // Chrome PDF viewer detected - use the page URL return { url: window.location.href }; } // Fallback to window.location.href anyway return { url: window.location.href }; })() """, 'returnByValue': True, }, session_id=temp_session.session_id, ), timeout=5.0, # 5 second timeout to prevent hanging ) pdf_info = result.get('result', {}).get('value', {}) pdf_url = pdf_info.get('url', '') if not pdf_url: self.logger.warning(f'[DownloadsWatchdog] ❌ Could not determine PDF URL for download {pdf_info}') return None # Generate filename from URL pdf_filename = os.path.basename(pdf_url.split('?')[0]) # Remove query params if not pdf_filename or not pdf_filename.endswith('.pdf'): parsed = urlparse(pdf_url) pdf_filename = os.path.basename(parsed.path) or 'document.pdf' if not pdf_filename.endswith('.pdf'): pdf_filename += '.pdf' self.logger.debug(f'[DownloadsWatchdog] Generated filename: {pdf_filename}') # Check if already downloaded by looking in the downloads directory downloads_dir = str(self.browser_session.browser_profile.downloads_path) if os.path.exists(downloads_dir): existing_files = os.listdir(downloads_dir) if pdf_filename in existing_files: self.logger.debug(f'[DownloadsWatchdog] PDF already downloaded: {pdf_filename}') return None self.logger.debug(f'[DownloadsWatchdog] Starting PDF download from: {pdf_url[:100]}...') # Download using JavaScript fetch to leverage browser cache try: # Properly escape the URL to prevent JavaScript injection escaped_pdf_url = json.dumps(pdf_url) result = await asyncio.wait_for( temp_session.cdp_client.send.Runtime.evaluate( params={ 'expression': f""" (async () => {{ try {{ // Use fetch with cache: 'force-cache' to prioritize cached version const response = await fetch({escaped_pdf_url}, {{ cache: 'force-cache' }}); if (!response.ok) {{ throw new Error(`HTTP error! status: ${{response.status}}`); }} const blob = await response.blob(); const arrayBuffer = await blob.arrayBuffer(); const uint8Array = new Uint8Array(arrayBuffer); // Check if served from cache const fromCache = response.headers.has('age') || !response.headers.has('date'); return {{ data: Array.from(uint8Array), fromCache: fromCache, responseSize: uint8Array.length, transferSize: response.headers.get('content-length') || 'unknown' }}; }} catch (error) {{ throw new Error(`Fetch failed: ${{error.message}}`); }} }})() """, 'awaitPromise': True, 'returnByValue': True, }, session_id=temp_session.session_id, ), timeout=10.0, # 10 second timeout for download operation ) download_result = result.get('result', {}).get('value', {}) if download_result and download_result.get('data') and len(download_result['data']) > 0: # Ensure unique filename downloads_dir = str(self.browser_session.browser_profile.downloads_path) # Ensure downloads directory exists os.makedirs(downloads_dir, exist_ok=True) unique_filename = await self._get_unique_filename(downloads_dir, pdf_filename) download_path = os.path.join(downloads_dir, unique_filename) # Save the PDF asynchronously async with await anyio.open_file(download_path, 'wb') as f: await f.write(bytes(download_result['data'])) # Verify file was written successfully if os.path.exists(download_path): actual_size = os.path.getsize(download_path) self.logger.debug( f'[DownloadsWatchdog] PDF file written successfully: {download_path} ({actual_size} bytes)' ) else: self.logger.error(f'[DownloadsWatchdog] ❌ Failed to write PDF file to: {download_path}') return None # Log cache information cache_status = 'from cache' if download_result.get('fromCache') else 'from network' response_size = download_result.get('responseSize', 0) self.logger.debug( f'[DownloadsWatchdog] ✅ Auto-downloaded PDF ({cache_status}, {response_size:,} bytes): {download_path}' ) # Emit file downloaded event self.logger.debug(f'[DownloadsWatchdog] Dispatching FileDownloadedEvent for {unique_filename}') self.event_bus.dispatch( FileDownloadedEvent( url=pdf_url, path=download_path, file_name=unique_filename, file_size=response_size, file_type='pdf', mime_type='application/pdf', from_cache=download_result.get('fromCache', False), auto_download=True, ) ) # No need to detach - session is cached return download_path else: self.logger.warning(f'[DownloadsWatchdog] No data received when downloading PDF from {pdf_url}') return None except Exception as e: self.logger.warning(f'[DownloadsWatchdog] Failed to auto-download PDF from {pdf_url}: {type(e).__name__}: {e}') return None except TimeoutError: self.logger.debug('[DownloadsWatchdog] PDF download operation timed out') return None except Exception as e: self.logger.error(f'[DownloadsWatchdog] Error in PDF download: {type(e).__name__}: {e}') return None @staticmethod async def _get_unique_filename(directory: str, filename: str) -> str: """Generate a unique filename for downloads by appending (1), (2), etc., if a file already exists.""" base, ext = os.path.splitext(filename) counter = 1 new_filename = filename while os.path.exists(os.path.join(directory, new_filename)): new_filename = f'{base} ({counter}){ext}' counter += 1 return new_filename # Fix Pydantic circular dependency - this will be called from session.py after BrowserSession is defined