ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter4/perception-tools/src/__init__.py

@@ -0,0 +1,7 @@
+"""
+Perception Tools MCP Server
+
+A comprehensive MCP server for perception and data retrieval capabilities.
+"""
+
+__version__ = "1.0.0"

chapter4/perception-tools/src/arxiv_enhanced.py

@@ -0,0 +1,209 @@
+"""
+Enhanced ArXiv tools with download and details.
+Based on AWorld parxiv-server complete implementation.
+"""
+import hashlib
+import json
+import logging
+import os
+import re
+import traceback
+from typing import Union
+
+import arxiv
+import httpx
+from dotenv import load_dotenv
+from mcp.types import TextContent
+
+from base import ActionResponse
+
+
+load_dotenv()
+
+
+async def get_paper_details(
+    paper_id: str
+) -> Union[str, TextContent]:
+    """
+    Get detailed information about an ArXiv paper.
+    
+    Args:
+        paper_id: ArXiv paper ID (e.g., '2301.07041')
+        
+    Returns:
+        TextContent with paper details
+    """
+    try:
+        clean_id = re.sub(r"^arxiv:", "", paper_id, flags=re.IGNORECASE).strip()
+        
+        logging.info(f"📄 Getting paper details: {clean_id}")
+        
+        search = arxiv.Search(id_list=[clean_id])
+        paper = next(arxiv.Client().results(search), None)
+        
+        if not paper:
+            raise ValueError(f"Paper not found: {clean_id}")
+        
+        result = {
+            "entry_id": paper.entry_id,
+            "title": paper.title,
+            "authors": [author.name for author in paper.authors],
+            "summary": paper.summary,
+            "published": paper.published.isoformat(),
+            "updated": paper.updated.isoformat() if paper.updated else None,
+            "categories": paper.categories,
+            "primary_category": paper.primary_category,
+            "pdf_url": paper.pdf_url,
+            "doi": paper.doi,
+            "journal_ref": paper.journal_ref
+        }
+        
+        logging.info(f"✅ Retrieved paper: {paper.title}")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"paper_id": clean_id}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Failed to get paper details: {str(e)}"
+        logging.error(f"ArXiv error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "arxiv_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def download_paper(
+    paper_id: str,
+    download_dir: str = "."
+) -> Union[str, TextContent]:
+    """
+    Download ArXiv paper PDF.
+    
+    Args:
+        paper_id: ArXiv paper ID
+        download_dir: Directory to save PDF
+        
+    Returns:
+        TextContent with download result
+    """
+    try:
+        from pathlib import Path
+        
+        clean_id = re.sub(r"^arxiv:", "", paper_id, flags=re.IGNORECASE).strip()
+        
+        logging.info(f"📥 Downloading paper: {clean_id}")
+        
+        if not re.fullmatch(
+            r"(?:[a-z-]+(?:\.[A-Z]{2})?/\d{7}|\d{4}\.\d{4,5})(?:v\d+)?",
+            clean_id,
+            flags=re.IGNORECASE,
+        ):
+            raise ValueError(f"Invalid arXiv paper ID: {clean_id}")
+
+        # Fetch the canonical PDF directly. Re-querying the Atom metadata API
+        # for every ID introduces an unrelated failure point and triggers its
+        # batch-query backoff during the three-paper experiment.
+        download_path = Path(download_dir)
+        download_path.mkdir(parents=True, exist_ok=True)
+        filename = f"{clean_id.replace('/', '_')}.pdf"
+        file_path = download_path / filename
+        temporary_path = download_path / f".{filename}.part"
+        pdf_url = f"https://arxiv.org/pdf/{clean_id}.pdf"
+        async with httpx.AsyncClient(
+            timeout=180,
+            follow_redirects=True,
+            headers={"User-Agent": "ai-agent-book-experiment/4.6"},
+        ) as client:
+            response = await client.get(pdf_url)
+            response.raise_for_status()
+            content = response.content
+        if len(content) <= 1000 or not content.startswith(b"%PDF-"):
+            raise ValueError("arXiv response was not a substantive PDF")
+        temporary_path.write_bytes(content)
+        os.replace(temporary_path, file_path)
+
+        result = {
+            "paper_id": clean_id,
+            "file_path": str(file_path),
+            "file_size": len(content),
+            "sha256": hashlib.sha256(content).hexdigest(),
+            "pdf_url": pdf_url,
+            "content_type": response.headers.get("content-type"),
+        }
+
+        logging.info(f"✅ Downloaded: {len(content)} bytes")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"paper_id": clean_id}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        action_response = ActionResponse(
+            success=False,
+            message=f"Download failed: {str(e)}",
+            metadata={"error_type": "download_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_arxiv_categories() -> Union[str, TextContent]:
+    """
+    Get list of ArXiv subject categories.
+    
+    Returns:
+        TextContent with categories
+    """
+    categories = {
+        "cs": "Computer Science",
+        "math": "Mathematics",
+        "physics": "Physics",
+        "astro-ph": "Astrophysics",
+        "cond-mat": "Condensed Matter",
+        "q-bio": "Quantitative Biology",
+        "q-fin": "Quantitative Finance",
+        "stat": "Statistics",
+        "econ": "Economics",
+        "eess": "Electrical Engineering"
+    }
+    
+    result = {
+        "categories": categories,
+        "count": len(categories)
+    }
+    
+    action_response = ActionResponse(
+        success=True,
+        message=result,
+        metadata={"total_categories": len(categories)}
+    )
+    
+    return TextContent(
+        type="text",
+        text=json.dumps(action_response.model_dump())
+    )

chapter4/perception-tools/src/base.py

@@ -0,0 +1,142 @@
+"""
+Base models and utilities for perception tools MCP server.
+"""
+import logging
+import os
+import tempfile
+import traceback
+from pathlib import Path
+from urllib.parse import urlparse
+from typing import Any
+
+import requests
+from pydantic import BaseModel, Field
+
+
+class ActionResponse(BaseModel):
+    """Standard response format for all perception tool actions."""
+    
+    success: bool = Field(default=False, description="Whether the action was successfully executed")
+    message: Any = Field(default=None, description="The execution result of the action")
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Additional metadata about the action")
+
+
+class DocumentMetadata(BaseModel):
+    """Metadata for document processing operations."""
+    
+    file_name: str = Field(description="Original file name")
+    file_size: int = Field(description="File size in bytes")
+    file_type: str = Field(description="Document file type/extension")
+    absolute_path: str = Field(description="Absolute path to the document file")
+    page_count: int | None = Field(default=None, description="Number of pages in document")
+    processing_time: float | None = Field(default=None, description="Time taken to process")
+    output_format: str = Field(description="Format of the extracted content")
+
+
+def is_url(path_or_url: str) -> bool:
+    """
+    Check if the given string is a URL.
+    
+    Args:
+        path_or_url: String to check
+        
+    Returns:
+        True if the string is a URL, False otherwise
+    """
+    parsed = urlparse(path_or_url)
+    return bool(parsed.scheme and parsed.netloc)
+
+
+def validate_file_path(file_path: str) -> Path:
+    """
+    Validate and resolve file path.
+    
+    Args:
+        file_path: Path to the file
+        
+    Returns:
+        Resolved Path object
+        
+    Raises:
+        FileNotFoundError: If file doesn't exist
+    """
+    path = Path(file_path).expanduser().resolve()
+    
+    if not path.exists():
+        raise FileNotFoundError(f"File not found: {path}")
+    
+    if not path.is_file():
+        raise ValueError(f"Path is not a file: {path}")
+    
+    return path
+
+
+def download_file_from_url(
+    url: str,
+    timeout: int = 60,
+    max_size_mb: float = 100.0
+) -> tuple[str, bytes]:
+    """
+    Download file from URL to temporary location.
+    
+    Args:
+        url: URL to download from
+        timeout: Request timeout in seconds
+        max_size_mb: Maximum file size in MB
+        
+    Returns:
+        Tuple of (temp_file_path, content)
+        
+    Raises:
+        ValueError: If file size exceeds limit
+        requests.RequestException: If download fails
+    """
+    max_size_bytes = max_size_mb * 1024 * 1024
+    
+    # Best-effort size pre-check. Many hosts refuse HEAD (presigned S3/GCS URLs
+    # sign the verb and return 403; CDN/WAF-fronted endpoints often return 405),
+    # so a failed HEAD must not abort a download that GET can serve -- the
+    # streaming loop below enforces max_size_bytes either way.
+    try:
+        head_response = requests.head(url, timeout=timeout, allow_redirects=True)
+        head_response.raise_for_status()
+        content_length = head_response.headers.get("content-length")
+    except requests.RequestException:
+        content_length = None
+
+    if content_length and int(content_length) > max_size_bytes:
+        raise ValueError(
+            f"File size ({int(content_length) / (1024 * 1024):.2f} MB) "
+            f"exceeds maximum allowed size ({max_size_mb} MB)"
+        )
+
+    try:
+        # Download the file
+        response = requests.get(url, timeout=timeout, stream=True)
+        response.raise_for_status()
+        
+        # Read content with size checking
+        content = b""
+        for chunk in response.iter_content(chunk_size=8192):
+            if len(content) + len(chunk) > max_size_bytes:
+                raise ValueError(f"File size exceeds maximum allowed size ({max_size_mb} MB)")
+            content += chunk
+        
+        # Create temporary file
+        parsed_url = urlparse(url)
+        filename = os.path.basename(parsed_url.path) or "downloaded_file"
+        suffix = Path(filename).suffix or ".tmp"
+        
+        with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:
+            temp_file.write(content)
+            temp_path = temp_file.name
+        
+        return temp_path, content
+        
+    except requests.RequestException as e:
+        raise requests.RequestException(f"Failed to download file from URL: {e}")
+    except ValueError:
+        # Documented in the docstring; must not be rewrapped as IOError.
+        raise
+    except Exception as e:
+        raise IOError(f"Error downloading file: {e}")

chapter4/perception-tools/src/document_processing_tools.py

@@ -0,0 +1,344 @@
+"""
+Document processing tools for PDF, DOCX, PPTX, CSV, TXT.
+Based on AWorld MCP server implementation.
+"""
+import json
+import logging
+import traceback
+from pathlib import Path
+from typing import Union, Dict, Any
+
+import pandas as pd
+from docx import Document
+from pptx import Presentation
+import PyPDF2
+from dotenv import load_dotenv
+from mcp.types import TextContent
+
+from base import ActionResponse, validate_file_path
+
+
+load_dotenv()
+
+
+async def extract_pdf_text(
+    file_path: str,
+    page_range: str | None = None
+) -> Union[str, TextContent]:
+    """
+    Extract text from PDF file.
+    
+    Args:
+        file_path: Path to PDF file
+        page_range: Optional page range (e.g., "1-5" or "1,3,5")
+        
+    Returns:
+        TextContent with extracted text
+    """
+    try:
+        path = validate_file_path(file_path)
+        
+        logging.info(f"📄 Extracting PDF: {path}")
+        
+        with open(path, 'rb') as file:
+            reader = PyPDF2.PdfReader(file)
+            total_pages = len(reader.pages)
+            
+            # Parse page range
+            if page_range:
+                pages_to_extract = parse_page_range(page_range, total_pages)
+            else:
+                pages_to_extract = range(total_pages)
+            
+            # Extract text
+            text_parts = []
+            for page_num in pages_to_extract:
+                if page_num < total_pages:
+                    page = reader.pages[page_num]
+                    text = page.extract_text()
+                    text_parts.append(f"--- Page {page_num + 1} ---\n{text}\n")
+            
+            full_text = "\n".join(text_parts)
+            
+            result = {
+                "file_name": path.name,
+                "file_type": "pdf",
+                "total_pages": total_pages,
+                "pages_extracted": len(pages_to_extract),
+                "text": full_text[:50000],  # Limit to 50k chars
+                "text_length": len(full_text),
+                "truncated": len(full_text) > 50000
+            }
+            
+            logging.info(f"✅ Extracted {len(pages_to_extract)} pages from PDF")
+            
+            action_response = ActionResponse(
+                success=True,
+                message=result,
+                metadata={"file_path": str(path), "pages": total_pages}
+            )
+            
+            return TextContent(
+                type="text",
+                text=json.dumps(action_response.model_dump())
+            )
+            
+    except Exception as e:
+        error_msg = f"PDF extraction failed: {str(e)}"
+        logging.error(f"PDF error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "pdf_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def extract_docx_content(
+    file_path: str
+) -> Union[str, TextContent]:
+    """
+    Extract content from DOCX file.
+    
+    Args:
+        file_path: Path to DOCX file
+        
+    Returns:
+        TextContent with extracted content
+    """
+    try:
+        path = validate_file_path(file_path)
+        
+        logging.info(f"📄 Extracting DOCX: {path}")
+        
+        doc = Document(path)
+        
+        # Extract paragraphs
+        paragraphs = [para.text for para in doc.paragraphs if para.text.strip()]
+        
+        # Extract tables
+        tables_data = []
+        for table in doc.tables:
+            table_data = []
+            for row in table.rows:
+                row_data = [cell.text for cell in row.cells]
+                table_data.append(row_data)
+            tables_data.append(table_data)
+        
+        full_text = "\n\n".join(paragraphs)
+        
+        result = {
+            "file_name": path.name,
+            "file_type": "docx",
+            "paragraphs": len(paragraphs),
+            "tables": len(tables_data),
+            "text": full_text[:50000],
+            "text_length": len(full_text),
+            "truncated": len(full_text) > 50000,
+            "tables_data": tables_data if tables_data else []
+        }
+        
+        logging.info(f"✅ Extracted DOCX: {len(paragraphs)} paragraphs, {len(tables_data)} tables")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"file_path": str(path)}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"DOCX extraction failed: {str(e)}"
+        logging.error(f"DOCX error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "docx_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def extract_pptx_content(
+    file_path: str
+) -> Union[str, TextContent]:
+    """
+    Extract content from PPTX file.
+    
+    Args:
+        file_path: Path to PPTX file
+        
+    Returns:
+        TextContent with extracted content
+    """
+    try:
+        path = validate_file_path(file_path)
+        
+        logging.info(f"📊 Extracting PPTX: {path}")
+        
+        prs = Presentation(path)
+        
+        slides_content = []
+        for slide_num, slide in enumerate(prs.slides, 1):
+            slide_text = []
+            for shape in slide.shapes:
+                if hasattr(shape, "text") and shape.text.strip():
+                    slide_text.append(shape.text)
+            
+            if slide_text:
+                slides_content.append({
+                    "slide_number": slide_num,
+                    "text": "\n".join(slide_text)
+                })
+        
+        full_text = "\n\n".join([f"=== Slide {s['slide_number']} ===\n{s['text']}" for s in slides_content])
+        
+        result = {
+            "file_name": path.name,
+            "file_type": "pptx",
+            "total_slides": len(prs.slides),
+            "slides_with_content": len(slides_content),
+            "text": full_text[:50000],
+            "text_length": len(full_text),
+            "truncated": len(full_text) > 50000,
+            "slides": slides_content
+        }
+        
+        logging.info(f"✅ Extracted PPTX: {len(prs.slides)} slides")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"file_path": str(path)}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"PPTX extraction failed: {str(e)}"
+        logging.error(f"PPTX error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "pptx_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def extract_csv_content(
+    file_path: str,
+    max_rows: int = 1000
+) -> Union[str, TextContent]:
+    """
+    Extract and parse CSV file content.
+    
+    Args:
+        file_path: Path to CSV file
+        max_rows: Maximum rows to read
+        
+    Returns:
+        TextContent with parsed CSV data
+    """
+    try:
+        path = validate_file_path(file_path)
+        
+        logging.info(f"📊 Parsing CSV: {path}")
+        
+        # Read CSV with pandas
+        df = pd.read_csv(path, nrows=max_rows)
+        
+        result = {
+            "file_name": path.name,
+            "file_type": "csv",
+            "rows": len(df),
+            "columns": len(df.columns),
+            "column_names": df.columns.tolist(),
+            "data": df.to_dict(orient="records"),
+            "preview": df.head(10).to_string(),
+            "truncated": len(df) == max_rows
+        }
+        
+        logging.info(f"✅ Parsed CSV: {len(df)} rows, {len(df.columns)} columns")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"file_path": str(path)}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"CSV parsing failed: {str(e)}"
+        logging.error(f"CSV error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "csv_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+def parse_page_range(page_range: str, total_pages: int) -> list[int]:
+    """
+    Parse page range string into list of page numbers.
+    
+    Args:
+        page_range: String like "1-5" or "1,3,5" or "1-3,7,9-11"
+        total_pages: Total number of pages
+        
+    Returns:
+        List of page numbers (0-indexed)
+    """
+    pages = []
+    
+    for part in page_range.split(","):
+        part = part.strip()
+        if not part:
+            # Trailing/duplicate commas (e.g. "1,3," or "1,,3") are common in
+            # LLM tool args; skip empty segments instead of int("").
+            continue
+        if "-" in part:
+            bounds = part.split("-")
+            if len(bounds) != 2 or not bounds[0] or not bounds[1]:
+                raise ValueError(f"Invalid page range segment: {part!r}")
+            start, end = int(bounds[0]), int(bounds[1])
+            # Clamp both ends: the caller's guard is `page_num < total_pages`,
+            # which a negative index passes, and reader.pages[-1] is the LAST
+            # page -- so an unclamped start silently returns the wrong page.
+            pages.extend(range(max(0, start - 1), min(end, total_pages)))
+        else:
+            page_num = int(part) - 1
+            if 0 <= page_num < total_pages:
+                pages.append(page_num)
+    
+    return sorted(set(pages))

chapter4/perception-tools/src/filesystem_tools.py

@@ -0,0 +1,581 @@
+"""File-system perception tools and tightly scoped mutation helpers.
+
+Read operations retain their historical behavior.  Move, copy, and delete are
+available only beneath the directory named by ``PERCEPTION_MUTATION_ROOT``.
+They reject absolute paths, traversal, symlinks, and the private quarantine
+directory.  Delete and overwrite are implemented as reversible quarantine
+moves so the experiment never has to destroy user data.
+"""
+import hashlib
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+import traceback
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Union
+
+from dotenv import load_dotenv
+from mcp.types import TextContent
+
+from base import ActionResponse, validate_file_path
+
+
+load_dotenv()
+
+
+MUTATION_ROOT_ENV = "PERCEPTION_MUTATION_ROOT"
+QUARANTINE_DIRECTORY = ".perception-trash"
+
+
+def _mutation_error(operation: str, exc: Exception) -> TextContent:
+    action_response = ActionResponse(
+        success=False,
+        message=f"Filesystem {operation} failed: {exc}",
+        metadata={
+            "operation": operation,
+            "error_type": type(exc).__name__,
+            "mutation_root_env": MUTATION_ROOT_ENV,
+        },
+    )
+    return TextContent(
+        type="text",
+        text=json.dumps(action_response.model_dump()),
+    )
+
+
+def _mutation_root() -> Path:
+    configured = os.getenv(MUTATION_ROOT_ENV, "").strip()
+    if not configured:
+        raise PermissionError(
+            f"{MUTATION_ROOT_ENV} must name an explicit experiment workspace"
+        )
+    root = Path(configured).expanduser().resolve(strict=True)
+    if not root.is_dir():
+        raise NotADirectoryError(f"Mutation root is not a directory: {root}")
+    return root
+
+
+def _relative_parts(value: str) -> tuple[str, ...]:
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError("Path must be a non-empty relative path")
+    path = Path(value)
+    if path.is_absolute():
+        raise PermissionError("Absolute paths are not allowed for filesystem mutations")
+    if ".." in path.parts:
+        raise PermissionError("Parent traversal is not allowed for filesystem mutations")
+    parts = tuple(part for part in path.parts if part not in {"", "."})
+    if not parts:
+        raise PermissionError("The mutation workspace root itself cannot be changed")
+    if parts[0] == QUARANTINE_DIRECTORY:
+        raise PermissionError("The filesystem quarantine is managed by the server")
+    return parts
+
+
+def _inside_root(root: Path, path: Path) -> bool:
+    try:
+        path.relative_to(root)
+        return True
+    except ValueError:
+        return False
+
+
+def _resolve_mutation_path(
+    root: Path,
+    value: str,
+    *,
+    must_exist: bool,
+) -> Path:
+    parts = _relative_parts(value)
+    unresolved = root.joinpath(*parts)
+    if must_exist:
+        resolved = unresolved.resolve(strict=True)
+    else:
+        parent = unresolved.parent.resolve(strict=True)
+        if not parent.is_dir():
+            raise NotADirectoryError(f"Destination parent is not a directory: {parent}")
+        resolved = parent / unresolved.name
+    if not _inside_root(root, resolved):
+        raise PermissionError("Resolved path escapes the configured mutation root")
+    if unresolved.is_symlink() or (resolved.exists() and resolved.is_symlink()):
+        raise PermissionError("Symbolic links are not allowed for filesystem mutations")
+    return resolved
+
+
+def _assert_no_symlinks(path: Path) -> None:
+    if path.is_symlink():
+        raise PermissionError(f"Symbolic links are not allowed: {path}")
+    if path.is_dir():
+        for item in path.rglob("*"):
+            if item.is_symlink():
+                raise PermissionError(f"Symbolic links are not allowed: {item}")
+
+
+def _fingerprint(path: Path) -> dict:
+    """Return a deterministic content receipt for one file or directory."""
+    digest = hashlib.sha256()
+    total_bytes = 0
+    entries = 0
+    if path.is_file():
+        with path.open("rb") as stream:
+            for block in iter(lambda: stream.read(1024 * 1024), b""):
+                digest.update(block)
+                total_bytes += len(block)
+        entries = 1
+        kind = "file"
+    elif path.is_dir():
+        kind = "directory"
+        for item in sorted(path.rglob("*"), key=lambda candidate: candidate.as_posix()):
+            if item.is_symlink():
+                raise PermissionError(f"Symbolic links are not allowed: {item}")
+            relative = item.relative_to(path).as_posix()
+            item_kind = "directory" if item.is_dir() else "file"
+            digest.update(f"{item_kind}\0{relative}\0".encode("utf-8"))
+            entries += 1
+            if item.is_file():
+                with item.open("rb") as stream:
+                    for block in iter(lambda: stream.read(1024 * 1024), b""):
+                        digest.update(block)
+                        total_bytes += len(block)
+    else:
+        raise ValueError(f"Unsupported filesystem object: {path}")
+    return {
+        "kind": kind,
+        "sha256": digest.hexdigest(),
+        "bytes": total_bytes,
+        "entries": entries,
+    }
+
+
+def _quarantine(root: Path, path: Path) -> Path:
+    trash = root / QUARANTINE_DIRECTORY
+    trash.mkdir(mode=0o700, exist_ok=True)
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
+    destination = trash / f"{stamp}-{uuid.uuid4().hex}-{path.name}"
+    path.rename(destination)
+    return destination
+
+
+async def move_path(
+    source_path: str,
+    destination_path: str,
+    overwrite: bool = False,
+) -> TextContent:
+    """Move a file/directory inside the explicit mutation workspace."""
+    operation = "move"
+    quarantined_destination = None
+    try:
+        root = _mutation_root()
+        source = _resolve_mutation_path(root, source_path, must_exist=True)
+        destination = _resolve_mutation_path(root, destination_path, must_exist=False)
+        if source == destination:
+            raise ValueError("Source and destination must be different")
+        _assert_no_symlinks(source)
+        before = _fingerprint(source)
+        if destination.exists():
+            if not overwrite:
+                raise FileExistsError(f"Destination already exists: {destination_path}")
+            _assert_no_symlinks(destination)
+            quarantined_destination = _quarantine(root, destination)
+        try:
+            source.rename(destination)
+        except Exception:
+            if quarantined_destination and not destination.exists():
+                quarantined_destination.rename(destination)
+            raise
+        after = _fingerprint(destination)
+        if before != after or source.exists():
+            raise RuntimeError("Post-move verification failed")
+        response = ActionResponse(
+            success=True,
+            message={
+                "operation": operation,
+                "source": source_path,
+                "destination": destination_path,
+                "source_exists_after": source.exists(),
+                "destination_fingerprint": after,
+                "replaced_path_quarantine": (
+                    str(quarantined_destination.relative_to(root))
+                    if quarantined_destination else None
+                ),
+            },
+            metadata={
+                "mutation_root": str(root),
+                "pre_operation_fingerprint": before,
+                "verification": "source absent and destination fingerprint matches",
+            },
+        )
+        return TextContent(type="text", text=json.dumps(response.model_dump()))
+    except Exception as exc:
+        logging.error("Filesystem move error: %s", traceback.format_exc())
+        return _mutation_error(operation, exc)
+
+
+async def copy_path(
+    source_path: str,
+    destination_path: str,
+    overwrite: bool = False,
+) -> TextContent:
+    """Copy a file/directory inside the explicit mutation workspace."""
+    operation = "copy"
+    quarantined_destination = None
+    try:
+        root = _mutation_root()
+        source = _resolve_mutation_path(root, source_path, must_exist=True)
+        destination = _resolve_mutation_path(root, destination_path, must_exist=False)
+        if source == destination:
+            raise ValueError("Source and destination must be different")
+        _assert_no_symlinks(source)
+        before = _fingerprint(source)
+        if destination.exists():
+            if not overwrite:
+                raise FileExistsError(f"Destination already exists: {destination_path}")
+            _assert_no_symlinks(destination)
+            quarantined_destination = _quarantine(root, destination)
+        try:
+            if source.is_dir():
+                shutil.copytree(source, destination, symlinks=False)
+            else:
+                shutil.copy2(source, destination)
+        except Exception:
+            if destination.exists():
+                if destination.is_dir():
+                    shutil.rmtree(destination)
+                else:
+                    destination.unlink()
+            if quarantined_destination:
+                quarantined_destination.rename(destination)
+            raise
+        after = _fingerprint(destination)
+        if before != after or not source.exists():
+            raise RuntimeError("Post-copy verification failed")
+        response = ActionResponse(
+            success=True,
+            message={
+                "operation": operation,
+                "source": source_path,
+                "destination": destination_path,
+                "source_exists_after": source.exists(),
+                "destination_fingerprint": after,
+                "replaced_path_quarantine": (
+                    str(quarantined_destination.relative_to(root))
+                    if quarantined_destination else None
+                ),
+            },
+            metadata={
+                "mutation_root": str(root),
+                "pre_operation_fingerprint": before,
+                "verification": "source retained and destination fingerprint matches",
+            },
+        )
+        return TextContent(type="text", text=json.dumps(response.model_dump()))
+    except Exception as exc:
+        logging.error("Filesystem copy error: %s", traceback.format_exc())
+        return _mutation_error(operation, exc)
+
+
+async def delete_path(path: str) -> TextContent:
+    """Remove a path from the workspace by moving it to private quarantine."""
+    operation = "delete"
+    try:
+        root = _mutation_root()
+        target = _resolve_mutation_path(root, path, must_exist=True)
+        _assert_no_symlinks(target)
+        before = _fingerprint(target)
+        quarantine = _quarantine(root, target)
+        after = _fingerprint(quarantine)
+        if target.exists() or before != after:
+            raise RuntimeError("Post-delete verification failed")
+        response = ActionResponse(
+            success=True,
+            message={
+                "operation": operation,
+                "path": path,
+                "path_exists_after": target.exists(),
+                "quarantine_path": str(quarantine.relative_to(root)),
+                "reversible": True,
+                "quarantine_fingerprint": after,
+            },
+            metadata={
+                "mutation_root": str(root),
+                "pre_operation_fingerprint": before,
+                "verification": "original path absent and quarantine fingerprint matches",
+            },
+        )
+        return TextContent(type="text", text=json.dumps(response.model_dump()))
+    except Exception as exc:
+        logging.error("Filesystem delete error: %s", traceback.format_exc())
+        return _mutation_error(operation, exc)
+
+
+async def read_file(
+    file_path: str,
+    encoding: str = "utf-8",
+    max_length: int = 50000
+) -> Union[str, TextContent]:
+    """
+    Read a file and return its contents.
+    
+    Args:
+        file_path: Path to the file
+        encoding: File encoding (default: utf-8)
+        max_length: Maximum number of characters to return
+        
+    Returns:
+        TextContent with file contents
+    """
+    try:
+        path = validate_file_path(file_path)
+        
+        logging.info(f"📖 Reading file: {path}")
+        
+        with open(path, 'r', encoding=encoding, errors='ignore') as f:
+            content = f.read()
+        
+        if max_length < 0:
+            max_length = len(content)
+        truncated = len(content) > max_length
+        if truncated:
+            content = content[:max_length]
+        
+        result = {
+            "file_path": str(path),
+            "content": content,
+            "size_bytes": path.stat().st_size,
+            "truncated": truncated,
+            "encoding": encoding
+        }
+        
+        logging.info(f"✅ Successfully read file ({len(content)} characters)")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"file_path": str(path)}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"File reading failed: {str(e)}"
+        logging.error(f"File read error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "file_read_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def grep_search(
+    pattern: str,
+    directory: str,
+    file_pattern: str = "*",
+    recursive: bool = True,
+    case_sensitive: bool = False,
+    max_results: int = 100
+) -> Union[str, TextContent]:
+    """
+    Search for a pattern in files using grep-like functionality.
+    
+    Args:
+        pattern: Regular expression pattern to search for
+        directory: Directory to search in
+        file_pattern: File pattern to match (e.g., "*.py")
+        recursive: Whether to search recursively
+        case_sensitive: Whether search is case-sensitive
+        max_results: Maximum number of results to return
+        
+    Returns:
+        TextContent with search results
+    """
+    try:
+        dir_path = Path(directory).expanduser().resolve()
+        
+        if not dir_path.exists():
+            raise FileNotFoundError(f"Directory not found: {dir_path}")
+        
+        if not dir_path.is_dir():
+            raise ValueError(f"Path is not a directory: {dir_path}")
+        
+        logging.info(f"🔍 Searching for pattern '{pattern}' in {dir_path}")
+        
+        results = []
+        if max_results <= 0:
+            action_response = ActionResponse(
+                success=True,
+                message={
+                    "pattern": pattern,
+                    "results": results,
+                    "total_found": 0,
+                    "truncated": False,
+                },
+                metadata={
+                    "directory": str(dir_path),
+                    "file_pattern": file_pattern,
+                    "recursive": recursive,
+                },
+            )
+            return TextContent(
+                type="text",
+                text=json.dumps(action_response.model_dump()),
+            )
+
+        flags = re.IGNORECASE if not case_sensitive else 0
+        regex = re.compile(pattern, flags)
+
+        if recursive:
+            files = dir_path.rglob(file_pattern)
+        else:
+            files = dir_path.glob(file_pattern)
+
+        for file_path in files:
+            if not file_path.is_file():
+                continue
+
+            try:
+                with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                    for line_num, line in enumerate(f, 1):
+                        if regex.search(line):
+                            results.append({
+                                "file": str(file_path.relative_to(dir_path)),
+                                "line_number": line_num,
+                                "line": line.strip(),
+                                "absolute_path": str(file_path)
+                            })
+
+                            if len(results) >= max_results:
+                                break
+
+                if len(results) >= max_results:
+                    break
+
+            except Exception as e:
+                logging.warning(f"Error reading {file_path}: {e}")
+                continue
+
+        logging.info(f"✅ Found {len(results)} matches")
+
+        action_response = ActionResponse(
+            success=True,
+            message={
+                "pattern": pattern,
+                "results": results,
+                "total_found": len(results),
+                "truncated": len(results) >= max_results
+            },
+
+            metadata={
+                "directory": str(dir_path),
+                "file_pattern": file_pattern,
+                "recursive": recursive
+            }
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Grep search failed: {str(e)}"
+        logging.error(f"Grep error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "grep_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def summarize_text(
+    text: str,
+    max_length: int = 500,
+    use_llm: bool = True
+) -> Union[str, TextContent]:
+    """
+    Summarize long text content.
+    
+    Args:
+        text: Text to summarize
+        max_length: Target summary length
+        use_llm: Whether to use LLM for summarization (if available)
+        
+    Returns:
+        TextContent with summary
+    """
+    try:
+        logging.info(f"📝 Summarizing text ({len(text)} characters)")
+        
+        if use_llm:
+            # TODO: Integrate with LLM API for better summarization
+            # For now, use simple extraction
+            summary = "LLM summarization not yet implemented. Using simple extraction."
+            method = "placeholder"
+        else:
+            # Simple extractive summarization: first N sentences
+            sentences = re.split(r'[.!?]+', text)
+            summary = ""
+            for sentence in sentences:
+                if len(summary) + len(sentence) > max_length:
+                    break
+                summary += sentence.strip() + ". "
+            method = "extractive"
+        
+        if not summary or summary == "LLM summarization not yet implemented. Using simple extraction.":
+            # Fallback: just truncate
+            summary = text[:max_length] + "..." if len(text) > max_length else text
+            method = "truncation"
+        
+        result = {
+            "original_length": len(text),
+            "summary_length": len(summary),
+            "summary": summary,
+            "method": method,
+            "compression_ratio": len(summary) / len(text) if len(text) > 0 else 0
+        }
+        
+        logging.info(f"✅ Generated summary ({len(summary)} characters)")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"method": method}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Text summarization failed: {str(e)}"
+        logging.error(f"Summarization error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "summarization_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )

chapter4/perception-tools/src/google_search_enhanced.py

@@ -0,0 +1,209 @@
+"""
+Enhanced Google Search tools.
+Based on AWorld google-search server.
+"""
+import json
+import logging
+import os
+import traceback
+from typing import Union
+
+import requests
+from bs4 import BeautifulSoup
+from dotenv import load_dotenv
+from mcp.types import TextContent
+
+from base import ActionResponse
+
+
+load_dotenv()
+
+
+async def google_search_api(
+    query: str,
+    num_results: int = 5,
+    safe_search: bool = True,
+    language: str = "en",
+    country: str = "us"
+) -> Union[str, TextContent]:
+    """
+    Search Google using Custom Search API.
+    
+    Args:
+        query: Search query
+        num_results: Number of results (1-10)
+        safe_search: Enable safe search
+        language: Language code
+        country: Country code
+        
+    Returns:
+        TextContent with search results
+    """
+    try:
+        api_key = os.getenv("GOOGLE_API_KEY")
+        cse_id = os.getenv("GOOGLE_CSE_ID")
+        
+        if not api_key or not cse_id:
+            return await _fallback_google_search(query, num_results)
+        
+        url = "https://www.googleapis.com/customsearch/v1"
+        params = {
+            "key": api_key,
+            "cx": cse_id,
+            "q": query,
+            "num": min(num_results, 10),
+            "safe": "active" if safe_search else "off",
+            "hl": language,
+            "gl": country
+        }
+        
+        response = requests.get(url, params=params, timeout=10)
+        response.raise_for_status()
+        data = response.json()
+        
+        results = []
+        if "items" in data:
+            for item in data["items"]:
+                results.append({
+                    "title": item.get("title"),
+                    "url": item.get("link"),
+                    "snippet": item.get("snippet"),
+                    "display_url": item.get("displayLink")
+                })
+        
+        action_response = ActionResponse(
+            success=True,
+            message={"query": query, "results": results, "count": len(results)},
+            metadata={"engine": "google_api", "results_count": len(results)}
+        )
+        
+        logging.info(f"✅ Google Search: {len(results)} results")
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        logging.error(f"Google API search failed: {traceback.format_exc()}")
+        return await _fallback_google_search(query, num_results)
+
+
+async def _fallback_google_search(query: str, num_results: int) -> TextContent:
+    """Fallback to DuckDuckGo if Google API not available."""
+    try:
+        logging.info("Using DuckDuckGo fallback")
+        
+        url = "https://html.duckduckgo.com/html/"
+        headers = {"User-Agent": "Mozilla/5.0"}
+        data = {"q": query}
+        
+        response = requests.post(url, data=data, headers=headers, timeout=15)
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.text, 'html.parser')
+        result_divs = soup.find_all('div', class_='result')
+        
+        results = []
+        for i, div in enumerate(result_divs[:num_results]):
+            title_tag = div.find('a', class_='result__a')
+            if title_tag:
+                results.append({
+                    "title": title_tag.get_text(strip=True),
+                    "url": title_tag.get('href', ''),
+                    "snippet": div.find('a', class_='result__snippet').get_text(strip=True) if div.find('a', class_='result__snippet') else ""
+                })
+        
+        action_response = ActionResponse(
+            success=True,
+            message={"query": query, "results": results, "count": len(results)},
+            metadata={"engine": "duckduckgo_fallback"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        action_response = ActionResponse(
+            success=False,
+            message=f"Search failed: {str(e)}",
+            metadata={"error_type": "search_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def read_webpage_content(
+    url: str,
+    extract_links: bool = False
+) -> Union[str, TextContent]:
+    """
+    Read and extract content from webpage.
+    
+    Args:
+        url: URL to read
+        extract_links: Whether to extract links
+        
+    Returns:
+        TextContent with webpage content
+    """
+    try:
+        headers = {"User-Agent": "Mozilla/5.0"}
+        response = requests.get(url, headers=headers, timeout=30)
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.content, 'html.parser')
+        
+        # Remove scripts and styles
+        for script in soup(["script", "style"]):
+            script.decompose()
+        
+        text = soup.get_text()
+        lines = (line.strip() for line in text.splitlines())
+        text = ' '.join(line for line in lines if line)
+        
+        result = {
+            "url": url,
+            "title": soup.title.string if soup.title else "No title",
+            "text": text[:10000],
+            "text_length": len(text)
+        }
+        
+        if extract_links:
+            links = []
+            for link in soup.find_all('a', href=True)[:50]:
+                links.append({
+                    "text": link.get_text().strip(),
+                    "href": link['href']
+                })
+            result["links"] = links
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"url": url}
+        )
+        
+        logging.info(f"✅ Read webpage: {len(text)} chars")
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        action_response = ActionResponse(
+            success=False,
+            message=f"Failed to read webpage: {str(e)}",
+            metadata={"error_type": "webpage_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )

chapter4/perception-tools/src/main.py

@@ -0,0 +1,704 @@
+"""
+Main MCP server for perception tools.
+
+This MCP server provides comprehensive perception capabilities including:
+- Search tools (web search, knowledge base, file download)
+- Multimodal understanding (web pages, documents, images, videos)
+- File system operations (read, grep, summarization)
+- Public data sources (weather, stocks, currency, Wikipedia, ArXiv, Wayback)
+- Private data sources (Google Calendar, Notion)
+"""
+import logging
+from dotenv import load_dotenv
+from mcp.server import MCPServer
+from pydantic import Field
+
+# Import all tool functions
+from search_tools import search_web, download_file, search_knowledge_base
+from multimodal_tools import read_webpage, read_document, parse_image, parse_video, extract_youtube_transcript, download_youtube_video
+from filesystem_tools import (
+    copy_path,
+    delete_path,
+    grep_search,
+    move_path,
+    read_file,
+    summarize_text,
+)
+from public_data_tools import (
+    get_weather, get_stock_price, convert_currency,
+    search_wikipedia, search_arxiv, search_wayback,
+    get_crypto_price, search_location, search_poi
+)
+from private_data_tools import get_calendar_events, search_notion
+from pubchem_tools import search_compounds, get_compound_properties, get_compound_synonyms, search_similar_compounds
+from yahoo_finance_tools import get_stock_quote, get_historical_data, get_company_info, get_financial_statements
+from document_processing_tools import extract_pdf_text, extract_docx_content, extract_pptx_content, extract_csv_content
+from media_processing_tools import transcribe_audio_whisper, extract_audio_metadata, extract_text_ocr, analyze_image_ai, extract_video_keyframes, analyze_video_ai, trim_audio, get_image_metadata
+from google_search_enhanced import google_search_api, read_webpage_content
+from wiki_enhanced import get_article_content, get_article_categories, get_article_links, get_article_history
+from arxiv_enhanced import get_paper_details, download_paper, get_arxiv_categories
+from wayback_enhanced import get_archived_content
+from expanded_catalog import enrich_existing_tools, register_expanded_tools
+
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+
+load_dotenv()
+
+# Initialize MCP server
+mcp = MCPServer(
+    "perception-tools",
+    instructions="""
+Perception Tools MCP Server
+
+A comprehensive MCP server providing various perception and data retrieval capabilities:
+
+## Search Tools
+- Web search using DuckDuckGo (free, no API key required)
+- Local knowledge base search
+- File download from URLs
+
+## Multimodal Understanding
+- Web page content extraction
+- Document reading (PDF, DOCX, PPTX)
+- Image parsing and analysis
+- Video metadata extraction
+
+## File System Tools
+- File reading with encoding support
+- Grep-like pattern search
+- Text summarization
+
+## Public Data Sources
+- Weather information
+- Stock prices and market data
+- Cryptocurrency prices (CoinGecko)
+- Currency conversion
+- Location search / geocoding (Nominatim)
+- Points of Interest search (Overpass)
+- Wikipedia search
+- ArXiv academic papers
+- Wayback Machine archives
+
+## Private Data Sources
+- Google Calendar events
+- Notion workspace search
+"""
+)
+
+
+# ============================================================================
+# SEARCH TOOLS
+# ============================================================================
+
+@mcp.tool(description="Search the web using DuckDuckGo (free, no API key required)")
+async def web_search(
+    query: str = Field(description="Search query string"),
+    num_results: int = Field(default=5, description="Number of results (1-10)"),
+    region: str = Field(default="wt-wt", description="Region code (e.g., 'us-en', 'uk-en', 'wt-wt' for worldwide)")
+):
+    """Search the web and return results."""
+    return await search_web(query, num_results, region)
+
+
+@mcp.tool(description="Download a file from a URL to local storage")
+async def download(
+    url: str = Field(description="URL to download from"),
+    output_path: str = Field(description="Local path to save the file"),
+    overwrite: bool = Field(default=False, description="Overwrite existing file"),
+    timeout: int = Field(default=180, description="Download timeout in seconds")
+):
+    """Download a file from URL."""
+    return await download_file(url, output_path, overwrite, timeout)
+
+
+@mcp.tool(description="Search a local knowledge base directory")
+async def knowledge_base_search(
+    query: str = Field(description="Search query"),
+    knowledge_base_path: str = Field(description="Path to knowledge base directory"),
+    top_k: int = Field(default=5, description="Number of top results")
+):
+    """Search local knowledge base."""
+    return await search_knowledge_base(query, knowledge_base_path, top_k)
+
+
+# ============================================================================
+# MULTIMODAL UNDERSTANDING TOOLS
+# ============================================================================
+
+@mcp.tool(description="Read and extract content from a webpage")
+async def webpage_reader(
+    url: str = Field(description="URL of the webpage"),
+    extract_text: bool = Field(default=True, description="Extract text content"),
+    extract_links: bool = Field(default=False, description="Extract links")
+):
+    """Read webpage content."""
+    return await read_webpage(url, extract_text, extract_links)
+
+
+@mcp.tool(description="Read and extract content from documents (PDF, DOCX, PPTX)")
+async def document_reader(
+    file_path: str = Field(description="Path to document file or URL"),
+    extract_images: bool = Field(default=False, description="Extract images")
+):
+    """Read document content."""
+    return await read_document(file_path, extract_images)
+
+
+@mcp.tool(description="Parse and analyze image files")
+async def image_parser(
+    image_path: str = Field(description="Path to image file or URL"),
+    use_llm: bool = Field(default=True, description="Use LLM for analysis")
+):
+    """Parse image content."""
+    return await parse_image(image_path, use_llm)
+
+
+@mcp.tool(description="Parse and extract metadata from video files")
+async def video_parser(
+    video_path: str = Field(description="Path to video file or URL"),
+    extract_frames: bool = Field(default=False, description="Extract sample frames"),
+    frame_interval: int = Field(default=30, description="Frame extraction interval")
+):
+    """Parse video metadata."""
+    return await parse_video(video_path, extract_frames, frame_interval)
+
+
+# ============================================================================
+# FILE SYSTEM TOOLS
+# ============================================================================
+
+@mcp.tool(description="Read a file and return its contents")
+async def file_reader(
+    file_path: str = Field(description="Path to the file"),
+    encoding: str = Field(default="utf-8", description="File encoding"),
+    max_length: int = Field(default=50000, description="Maximum characters to read")
+):
+    """Read file contents."""
+    return await read_file(file_path, encoding, max_length)
+
+
+@mcp.tool(description="Search for patterns in files (grep-like functionality)")
+async def grep(
+    pattern: str = Field(description="Regular expression pattern"),
+    directory: str = Field(description="Directory to search in"),
+    file_pattern: str = Field(default="*", description="File pattern (e.g., *.py)"),
+    recursive: bool = Field(default=True, description="Search recursively"),
+    case_sensitive: bool = Field(default=False, description="Case-sensitive search"),
+    max_results: int = Field(default=100, description="Maximum results")
+):
+    """Search files for pattern."""
+    return await grep_search(pattern, directory, file_pattern, recursive, case_sensitive, max_results)
+
+
+@mcp.tool(description="Summarize long text content")
+async def text_summarizer(
+    text: str = Field(description="Text to summarize"),
+    max_length: int = Field(default=500, description="Target summary length"),
+    use_llm: bool = Field(default=True, description="Use LLM for summarization")
+):
+    """Summarize text."""
+    return await summarize_text(text, max_length, use_llm)
+
+
+@mcp.tool(description="Move a file or directory inside the configured mutation workspace")
+async def filesystem_move(
+    source_path: str = Field(description="Relative source path beneath PERCEPTION_MUTATION_ROOT"),
+    destination_path: str = Field(description="Relative destination path beneath PERCEPTION_MUTATION_ROOT"),
+    overwrite: bool = Field(default=False, description="Quarantine an existing destination before moving"),
+):
+    """Move one workspace-confined filesystem object."""
+    return await move_path(source_path, destination_path, overwrite)
+
+
+@mcp.tool(description="Copy a file or directory inside the configured mutation workspace")
+async def filesystem_copy(
+    source_path: str = Field(description="Relative source path beneath PERCEPTION_MUTATION_ROOT"),
+    destination_path: str = Field(description="Relative destination path beneath PERCEPTION_MUTATION_ROOT"),
+    overwrite: bool = Field(default=False, description="Quarantine an existing destination before copying"),
+):
+    """Copy one workspace-confined filesystem object."""
+    return await copy_path(source_path, destination_path, overwrite)
+
+
+@mcp.tool(description="Delete a file or directory from the configured mutation workspace using reversible quarantine")
+async def filesystem_delete(
+    path: str = Field(description="Relative path beneath PERCEPTION_MUTATION_ROOT"),
+):
+    """Quarantine one workspace-confined filesystem object."""
+    return await delete_path(path)
+
+
+# ============================================================================
+# PUBLIC DATA SOURCE TOOLS
+# ============================================================================
+
+@mcp.tool(description="Get current weather information for a location (Open-Meteo, free, no API key)")
+async def weather(
+    location: str = Field(description="City name (automatically geocoded)"),
+    latitude: float | None = Field(default=None, description="Latitude coordinate (optional)"),
+    longitude: float | None = Field(default=None, description="Longitude coordinate (optional)")
+):
+    """Get weather data."""
+    return await get_weather(location, latitude, longitude)
+
+
+@mcp.tool(description="Get stock price and market information")
+async def stock_price(
+    symbol: str = Field(description="Stock ticker symbol (e.g., AAPL)"),
+    interval: str = Field(default="1d", description="Data interval")
+):
+    """Get stock price."""
+    return await get_stock_price(symbol, interval)
+
+
+@mcp.tool(description="Convert between currencies")
+async def currency_converter(
+    amount: float = Field(description="Amount to convert"),
+    from_currency: str = Field(description="Source currency code (e.g., USD)"),
+    to_currency: str = Field(description="Target currency code (e.g., EUR)")
+):
+    """Convert currency."""
+    return await convert_currency(amount, from_currency, to_currency)
+
+
+@mcp.tool(description="Get cryptocurrency price information (CoinGecko, free, no API key)")
+async def crypto_price(
+    symbol: str = Field(description="Cryptocurrency symbol or ID (e.g., bitcoin, ethereum, btc, eth)"),
+    vs_currency: str = Field(default="usd", description="Target currency (usd, eur, gbp, etc.)")
+):
+    """Get cryptocurrency price."""
+    return await get_crypto_price(symbol, vs_currency)
+
+
+@mcp.tool(description="Search for locations using Nominatim/OpenStreetMap (free, no API key)")
+async def location_search(
+    query: str = Field(description="Location query (e.g., 'Eiffel Tower', 'New York', 'Tokyo')"),
+    limit: int = Field(default=5, description="Maximum number of results (1-50)"),
+    country_code: str | None = Field(default=None, description="Country code filter (e.g., 'us', 'gb', 'fr')")
+):
+    """Search locations (geocoding)."""
+    return await search_location(query, limit, country_code)
+
+
+@mcp.tool(description="Search for Points of Interest near a location using Overpass/OpenStreetMap (free, no API key)")
+async def poi_search(
+    query: str = Field(description="Type of POI (e.g., 'restaurant', 'cafe', 'hospital', 'atm', 'hotel')"),
+    latitude: float = Field(description="Center latitude coordinate"),
+    longitude: float = Field(description="Center longitude coordinate"),
+    radius: int = Field(default=1000, description="Search radius in meters"),
+    limit: int = Field(default=10, description="Maximum number of results")
+):
+    """Search points of interest."""
+    return await search_poi(query, latitude, longitude, radius, limit)
+
+
+@mcp.tool(description="Search Wikipedia and get article summary")
+async def wikipedia_search(
+    query: str = Field(description="Search query"),
+    language: str = Field(default="en", description="Wikipedia language"),
+    sentences: int = Field(default=5, description="Summary sentence count")
+):
+    """Search Wikipedia."""
+    return await search_wikipedia(query, language, sentences)
+
+
+@mcp.tool(description="Search ArXiv for academic papers")
+async def arxiv_search(
+    query: str = Field(description="Search query"),
+    max_results: int = Field(default=5, description="Maximum results"),
+    sort_by: str = Field(default="relevance", description="Sort method")
+):
+    """Search ArXiv."""
+    return await search_arxiv(query, max_results, sort_by)
+
+
+@mcp.tool(description="Search Wayback Machine for archived web pages")
+async def wayback_search(
+    url: str = Field(description="URL to search for"),
+    year: int | None = Field(default=None, description="Filter by year"),
+    limit: int = Field(default=10, description="Maximum snapshots")
+):
+    """Search Wayback Machine."""
+    return await search_wayback(url, year, limit)
+
+
+# ============================================================================
+# YOUTUBE TOOLS
+# ============================================================================
+
+@mcp.tool(description="Extract transcript from a YouTube video")
+async def youtube_transcript(
+    video_id: str = Field(description="YouTube video ID or URL"),
+    language_code: str = Field(default="en", description="Language code for transcript"),
+    translate_to_language: str | None = Field(default=None, description="Translate to this language")
+):
+    """Extract YouTube transcript."""
+    return await extract_youtube_transcript(video_id, language_code, translate_to_language)
+
+
+# ============================================================================
+# PUBCHEM CHEMICAL DATA TOOLS
+# ============================================================================
+
+@mcp.tool(description="Search PubChem for chemical compounds")
+async def pubchem_search(
+    query: str = Field(description="Search term or identifier"),
+    search_type: str = Field(default="name", description="Type: name, cid, smiles, inchi, formula"),
+    max_results: int = Field(default=10, description="Maximum results (1-100)")
+):
+    """Search PubChem compounds."""
+    return await search_compounds(query, search_type, max_results)
+
+
+@mcp.tool(description="Get detailed properties for a PubChem compound")
+async def pubchem_properties(
+    cid: int = Field(description="PubChem Compound ID"),
+    properties: list[str] | None = Field(default=None, description="List of property names")
+):
+    """Get compound properties."""
+    return await get_compound_properties(cid, properties)
+
+
+@mcp.tool(description="Get synonyms for a PubChem compound")
+async def pubchem_synonyms(
+    cid: int = Field(description="PubChem Compound ID"),
+    max_synonyms: int = Field(default=20, description="Maximum synonyms (1-100)")
+):
+    """Get compound synonyms."""
+    return await get_compound_synonyms(cid, max_synonyms)
+
+
+@mcp.tool(description="Search for structurally similar compounds in PubChem")
+async def pubchem_similar(
+    cid: int = Field(description="Reference compound CID"),
+    similarity_threshold: float = Field(default=0.9, description="Similarity threshold (0.0-1.0)"),
+    max_results: int = Field(default=10, description="Maximum results (1-50)")
+):
+    """Search similar compounds."""
+    return await search_similar_compounds(cid, similarity_threshold, max_results)
+
+
+# ============================================================================
+# YAHOO FINANCE TOOLS
+# ============================================================================
+
+@mcp.tool(description="Get current stock quote and market data")
+async def yfinance_quote(
+    symbol: str = Field(description="Stock ticker symbol (e.g., AAPL, MSFT)")
+):
+    """Get stock quote."""
+    return await get_stock_quote(symbol)
+
+
+@mcp.tool(description="Get historical stock price data")
+async def yfinance_historical(
+    symbol: str = Field(description="Stock ticker symbol"),
+    start: str = Field(description="Start date (YYYY-MM-DD)"),
+    end: str = Field(description="End date (YYYY-MM-DD)"),
+    interval: str = Field(default="1d", description="Data interval (1d, 1wk, 1mo)"),
+    max_rows_preview: int = Field(default=10, description="Max rows in preview")
+):
+    """Get historical stock data."""
+    return await get_historical_data(symbol, start, end, interval, max_rows_preview)
+
+
+@mcp.tool(description="Get comprehensive company information")
+async def yfinance_company_info(
+    symbol: str = Field(description="Stock ticker symbol")
+):
+    """Get company information."""
+    return await get_company_info(symbol)
+
+
+@mcp.tool(description="Get financial statements (income statement, balance sheet, cash flow)")
+async def yfinance_financials(
+    symbol: str = Field(description="Stock ticker symbol"),
+    statement_type: str = Field(description="Type: income_statement, balance_sheet, cash_flow"),
+    period_type: str = Field(default="annual", description="Period: annual or quarterly"),
+    max_columns_preview: int = Field(default=4, description="Max periods to show")
+):
+    """Get financial statements."""
+    return await get_financial_statements(symbol, statement_type, period_type, max_columns_preview)
+
+
+# ============================================================================
+# DOCUMENT PROCESSING TOOLS
+# ============================================================================
+
+@mcp.tool(description="Extract text from PDF file with optional page range")
+async def pdf_extract(
+    file_path: str = Field(description="Path to PDF file"),
+    page_range: str | None = Field(default=None, description="Page range (e.g., '1-5' or '1,3,5')")
+):
+    """Extract text from PDF."""
+    return await extract_pdf_text(file_path, page_range)
+
+
+@mcp.tool(description="Extract content from Word document (DOCX)")
+async def docx_extract(
+    file_path: str = Field(description="Path to DOCX file")
+):
+    """Extract content from DOCX."""
+    return await extract_docx_content(file_path)
+
+
+@mcp.tool(description="Extract content from PowerPoint presentation (PPTX)")
+async def pptx_extract(
+    file_path: str = Field(description="Path to PPTX file")
+):
+    """Extract content from PPTX."""
+    return await extract_pptx_content(file_path)
+
+
+@mcp.tool(description="Extract and parse CSV file data")
+async def csv_parse(
+    file_path: str = Field(description="Path to CSV file"),
+    max_rows: int = Field(default=1000, description="Maximum rows to read")
+):
+    """Parse CSV data."""
+    return await extract_csv_content(file_path, max_rows)
+
+
+# ============================================================================
+# MEDIA PROCESSING TOOLS
+# ============================================================================
+
+@mcp.tool(description="Transcribe audio to text using Whisper")
+async def audio_transcribe(
+    file_path: str = Field(description="Path to audio file"),
+    model_size: str = Field(default="base", description="Whisper model size"),
+    language: str = Field(default="en", description="Language code")
+):
+    """Transcribe audio to text."""
+    return await transcribe_audio_whisper(file_path, model_size, language)
+
+
+@mcp.tool(description="Extract audio file metadata")
+async def audio_metadata(
+    file_path: str = Field(description="Path to audio file")
+):
+    """Extract audio metadata."""
+    return await extract_audio_metadata(file_path)
+
+
+@mcp.tool(description="Extract text from image using OCR")
+async def image_ocr(
+    image_path: str = Field(description="Path to image file"),
+    language: str = Field(default="eng", description="OCR language")
+):
+    """Extract text from image using OCR."""
+    return await extract_text_ocr(image_path, language)
+
+
+@mcp.tool(description="Analyze image using AI vision")
+async def image_analyze(
+    image_path: str = Field(description="Path to image file"),
+    prompt: str = Field(default="Describe this image in detail", description="Analysis prompt")
+):
+    """Analyze image with AI."""
+    return await analyze_image_ai(image_path, prompt)
+
+
+@mcp.tool(description="Extract keyframes from video")
+async def video_keyframes(
+    video_path: str = Field(description="Path to video file"),
+    num_frames: int = Field(default=10, description="Number of keyframes to extract")
+):
+    """Extract video keyframes."""
+    return await extract_video_keyframes(video_path, num_frames)
+
+
+@mcp.tool(description="Analyze video content using AI vision")
+async def video_analyze(
+    video_path: str = Field(description="Path to video file"),
+    num_frames: int = Field(default=5, description="Number of frames to analyze"),
+    prompt: str = Field(default="Analyze this video and describe what's happening", description="Analysis prompt")
+):
+    """Analyze video with AI."""
+    return await analyze_video_ai(video_path, num_frames, prompt)
+
+
+@mcp.tool(description="Trim audio file to specific time range")
+async def audio_trim(
+    audio_path: str = Field(description="Path to audio file"),
+    start_time: float = Field(description="Start time in seconds"),
+    duration: float | None = Field(default=None, description="Duration in seconds"),
+    output_path: str | None = Field(default=None, description="Output file path")
+):
+    """Trim audio file."""
+    return await trim_audio(audio_path, start_time, duration, output_path)
+
+
+@mcp.tool(description="Get detailed image metadata including EXIF")
+async def image_metadata(
+    image_path: str = Field(description="Path to image file")
+):
+    """Get image metadata."""
+    return await get_image_metadata(image_path)
+
+
+@mcp.tool(description="Download YouTube video")
+async def youtube_download(
+    url: str = Field(description="YouTube video URL"),
+    output_dir: str = Field(default=".", description="Output directory"),
+    max_resolution: str = Field(default="720p", description="Maximum resolution")
+):
+    """Download YouTube video."""
+    return await download_youtube_video(url, output_dir, max_resolution)
+
+
+# ============================================================================
+# GOOGLE SEARCH ENHANCED TOOLS
+# ============================================================================
+
+@mcp.tool(description="Search Google with API or DuckDuckGo fallback")
+async def google_search_enhanced(
+    query: str = Field(description="Search query"),
+    num_results: int = Field(default=5, description="Number of results (1-10)"),
+    safe_search: bool = Field(default=True, description="Enable safe search"),
+    language: str = Field(default="en", description="Language code"),
+    country: str = Field(default="us", description="Country code")
+):
+    """Enhanced Google search."""
+    return await google_search_api(query, num_results, safe_search, language, country)
+
+
+@mcp.tool(description="Read and extract content from webpage")
+async def webpage_read_enhanced(
+    url: str = Field(description="URL to read"),
+    extract_links: bool = Field(default=False, description="Extract links from page")
+):
+    """Read webpage content."""
+    return await read_webpage_content(url, extract_links)
+
+
+# ============================================================================
+# WIKIPEDIA ENHANCED TOOLS
+# ============================================================================
+
+@mcp.tool(description="Get full Wikipedia article content")
+async def wiki_article_full(
+    title: str = Field(description="Article title"),
+    language: str = Field(default="en", description="Language code")
+):
+    """Get full Wikipedia article."""
+    return await get_article_content(title, language)
+
+
+@mcp.tool(description="Get Wikipedia article categories")
+async def wiki_article_categories(
+    title: str = Field(description="Article title"),
+    language: str = Field(default="en", description="Language code")
+):
+    """Get article categories."""
+    return await get_article_categories(title, language)
+
+
+@mcp.tool(description="Get links from Wikipedia article")
+async def wiki_article_links(
+    title: str = Field(description="Article title"),
+    language: str = Field(default="en", description="Language code")
+):
+    """Get article links."""
+    return await get_article_links(title, language)
+
+
+@mcp.tool(description="Get historical version of Wikipedia article")
+async def wiki_article_history(
+    title: str = Field(description="Article title"),
+    date: str = Field(description="Date (YYYY/MM/DD)"),
+    language: str = Field(default="en", description="Language code")
+):
+    """Get historical Wikipedia article."""
+    return await get_article_history(title, date, language)
+
+
+# ============================================================================
+# ARXIV ENHANCED TOOLS
+# ============================================================================
+
+@mcp.tool(description="Get detailed ArXiv paper information")
+async def arxiv_paper_details(
+    paper_id: str = Field(description="ArXiv paper ID")
+):
+    """Get paper details."""
+    return await get_paper_details(paper_id)
+
+
+@mcp.tool(description="Download ArXiv paper PDF")
+async def arxiv_download(
+    paper_id: str = Field(description="ArXiv paper ID"),
+    download_dir: str = Field(default=".", description="Download directory")
+):
+    """Download ArXiv paper."""
+    return await download_paper(paper_id, download_dir)
+
+
+@mcp.tool(description="Get ArXiv subject categories")
+async def arxiv_categories():
+    """Get ArXiv categories."""
+    return await get_arxiv_categories()
+
+
+# ============================================================================
+# WAYBACK ENHANCED TOOLS
+# ============================================================================
+
+@mcp.tool(description="Get content from archived webpage")
+async def wayback_archived_content(
+    url: str = Field(description="URL to retrieve"),
+    timestamp: str = Field(description="Timestamp (YYYYMMDDhhmmss)")
+):
+    """Get archived webpage content."""
+    return await get_archived_content(url, timestamp)
+
+
+# ============================================================================
+# PRIVATE DATA SOURCE TOOLS
+# ============================================================================
+
+@mcp.tool(description="Get events from Google Calendar")
+async def calendar_events(
+    start_date: str | None = Field(default=None, description="Start date (ISO format)"),
+    end_date: str | None = Field(default=None, description="End date (ISO format)"),
+    calendar_id: str = Field(default="primary", description="Calendar ID"),
+    max_results: int = Field(default=10, description="Maximum events")
+):
+    """Get calendar events."""
+    return await get_calendar_events(start_date, end_date, calendar_id, max_results)
+
+
+@mcp.tool(description="Search Notion workspace")
+async def notion_search(
+    query: str = Field(description="Search query"),
+    database_id: str | None = Field(default=None, description="Specific database ID"),
+    page_size: int = Field(default=10, description="Results per page")
+):
+    """Search Notion."""
+    return await search_notion(query, database_id, page_size)
+
+
+# Complete the 56 native schema descriptions before adding the expanded
+# catalog. The implementations and native parameter schemas remain unchanged.
+enrich_existing_tools(mcp)
+
+# Experiment 4-7 requires 120+ tools from this perception MCP server.  The
+# 56 native tools plus 70 additional real-backed, read-mostly tools bring the
+# server catalog to 126 tools. Registration is dynamic only to avoid repetitive
+# wrapper functions; tools/list still
+# returns ordinary full JSON schemas and every tool is callable over MCP.
+register_expanded_tools(mcp)
+
+
+# ============================================================================
+# RUN SERVER
+# ============================================================================
+
+if __name__ == "__main__":
+    logging.info("Starting Perception Tools MCP server!")
+    mcp.run(transport="stdio")

chapter4/perception-tools/src/media_processing_tools.py

@@ -0,0 +1,858 @@
+"""
+Media processing tools for audio, image, and video.
+Based on AWorld MCP server implementation.
+"""
+import json
+import logging
+import traceback
+import subprocess
+import base64
+import os
+import time
+from pathlib import Path
+from typing import Union, Dict, Any
+
+import cv2
+from PIL import Image
+from dotenv import load_dotenv
+from mcp.types import TextContent
+
+from base import ActionResponse, validate_file_path
+
+
+load_dotenv()
+
+
+def _map_model_for_openrouter(model: str) -> str:
+    """Map a plain model id onto OpenRouter's `provider/model` form."""
+    if "/" in model:
+        return model
+    m = model.lower()
+    if m.startswith(("gpt-", "o1-", "o3-", "o4-")):
+        return f"openai/{model}"
+    if m.startswith("claude-"):
+        return "anthropic/claude-opus-4.8"
+    return model
+
+
+def _make_vision_client(default_model: str = "gpt-5.6-luna"):
+    """Build an OpenAI-compatible vision client with a universal fallback.
+
+    Preferred path uses OPENAI_API_KEY directly. When it is absent but an
+    OPENROUTER_API_KEY is set, transparently route through OpenRouter (mapping
+    the model id to provider/model form) so the vision tools still run.
+
+    Returns (client, model). Raises ValueError with the accepted keys listed
+    when neither credential is available.
+    """
+    import os
+    from openai import OpenAI
+
+    provider = os.getenv("PERCEPTION_VISION_PROVIDER", "").strip().lower()
+    if provider == "dashscope":
+        dashscope_key = os.getenv("DASHSCOPE_API_KEY")
+        if not dashscope_key:
+            raise ValueError(
+                "PERCEPTION_VISION_PROVIDER=dashscope requires DASHSCOPE_API_KEY"
+            )
+        client = OpenAI(
+            api_key=dashscope_key,
+            # The provided project credential is issued for the international
+            # Model Studio region; regional keys are not interchangeable.
+            base_url=os.getenv(
+                "DASHSCOPE_BASE_URL",
+                "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
+            ),
+            timeout=120.0,
+            max_retries=0,
+        )
+        return client, os.getenv("PERCEPTION_VISION_MODEL", "qwen-vl-max")
+    gemini_key = os.getenv("GEMINI_API_KEY")
+    if provider == "gemini":
+        if not gemini_key:
+            raise ValueError(
+                "PERCEPTION_VISION_PROVIDER=gemini requires GEMINI_API_KEY"
+            )
+        client = OpenAI(
+            api_key=gemini_key,
+            base_url="https://generativelanguage.googleapis.com/v1beta/openai/",
+        )
+        model = os.getenv("PERCEPTION_VISION_MODEL", "gemini-2.5-flash")
+        return client, model
+
+    model = os.getenv("PERCEPTION_VISION_MODEL", default_model)
+    or_key = os.getenv("OPENROUTER_API_KEY")
+    # gpt-5.x (incl. gpt-5.6*) needs OpenAI org-verification on the direct API;
+    # when an OpenRouter key is present, prefer routing these ids through it.
+    if or_key and model.lower().startswith("gpt-5"):
+        client = OpenAI(api_key=or_key, base_url="https://openrouter.ai/api/v1")
+        return client, _map_model_for_openrouter(model)
+
+    api_key = os.getenv("OPENAI_API_KEY")
+    if api_key:
+        base_url = os.getenv("OPENAI_BASE_URL")
+        client = OpenAI(api_key=api_key, base_url=base_url) if base_url else OpenAI(api_key=api_key)
+        return client, model
+
+    if or_key:
+        client = OpenAI(api_key=or_key, base_url="https://openrouter.ai/api/v1")
+        return client, _map_model_for_openrouter(model)
+
+    if gemini_key:
+        client = OpenAI(
+            api_key=gemini_key,
+            base_url="https://generativelanguage.googleapis.com/v1beta/openai/",
+        )
+        return client, os.getenv("PERCEPTION_VISION_MODEL", "gemini-2.5-flash")
+
+    raise ValueError(
+        "No vision key configured. Set OPENAI_API_KEY, OPENROUTER_API_KEY, or GEMINI_API_KEY."
+    )
+
+
+async def transcribe_audio_whisper(
+    file_path: str,
+    model_size: str = "base",
+    language: str = "en"
+) -> Union[str, TextContent]:
+    """
+    Transcribe audio file using OpenAI Whisper (local).
+    Note: Requires whisper package installed.
+    
+    Args:
+        file_path: Path to audio file
+        model_size: Whisper model size (tiny, base, small, medium, large)
+        language: Language code
+        
+    Returns:
+        TextContent with transcription
+    """
+    try:
+        path = validate_file_path(file_path)
+        
+        logging.info(f"🎤 Transcribing audio: {path}")
+        
+        try:
+            import whisper
+            
+            # Load model
+            model = whisper.load_model(model_size)
+            
+            # Transcribe
+            result = model.transcribe(str(path), language=language)
+            
+            transcription = result["text"]
+            
+            response_data = {
+                "file_name": path.name,
+                "file_type": path.suffix,
+                "model": model_size,
+                "language": language,
+                "transcription": transcription,
+                "word_count": len(transcription.split())
+            }
+            
+            logging.info(f"✅ Transcribed: {len(transcription)} chars")
+            
+            action_response = ActionResponse(
+                success=True,
+                message=response_data,
+                metadata={"file_path": str(path)}
+            )
+            
+        except ImportError:
+            # Fallback: try using OpenAI API if available
+            import os
+            from openai import OpenAI
+            
+            api_key = os.getenv("OPENAI_API_KEY")
+            if not api_key:
+                raise ImportError("Whisper not installed and no OPENAI_API_KEY found")
+            
+            client = OpenAI(api_key=api_key)
+            
+            with open(path, "rb") as audio_file:
+                transcription = client.audio.transcriptions.create(
+                    model="whisper-1",
+                    file=audio_file,
+                    language=language
+                )
+            
+            response_data = {
+                "file_name": path.name,
+                "file_type": path.suffix,
+                "model": "whisper-1 (API)",
+                "language": language,
+                "transcription": transcription.text,
+                "word_count": len(transcription.text.split())
+            }
+            
+            action_response = ActionResponse(
+                success=True,
+                message=response_data,
+                metadata={"file_path": str(path), "method": "openai_api"}
+            )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Audio transcription failed: {str(e)}"
+        logging.error(f"Audio error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "audio_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def extract_audio_metadata(
+    file_path: str
+) -> Union[str, TextContent]:
+    """
+    Extract audio file metadata using ffprobe.
+    
+    Args:
+        file_path: Path to audio file
+        
+    Returns:
+        TextContent with audio metadata
+    """
+    try:
+        path = validate_file_path(file_path)
+        
+        logging.info(f"🎵 Extracting audio metadata: {path}")
+        
+        # Use ffprobe to get metadata
+        cmd = [
+            "ffprobe",
+            "-v", "quiet",
+            "-print_format", "json",
+            "-show_format",
+            "-show_streams",
+            str(path)
+        ]
+        
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+        
+        if result.returncode == 0:
+            metadata = json.loads(result.stdout)
+            
+            format_info = metadata.get("format", {})
+            streams = metadata.get("streams", [])
+            audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), {})
+            
+            response_data = {
+                "file_name": path.name,
+                "file_size": path.stat().st_size,
+                "duration": float(format_info.get("duration", 0)),
+                "bit_rate": int(format_info.get("bit_rate", 0)),
+                "format": format_info.get("format_name"),
+                "codec": audio_stream.get("codec_name"),
+                "sample_rate": int(audio_stream.get("sample_rate", 0)) if audio_stream.get("sample_rate") else None,
+                "channels": int(audio_stream.get("channels", 0)) if audio_stream.get("channels") else None
+            }
+            
+            logging.info(f"✅ Audio metadata extracted")
+            
+            action_response = ActionResponse(
+                success=True,
+                message=response_data,
+                metadata={"file_path": str(path)}
+            )
+        else:
+            raise RuntimeError(f"ffprobe failed: {result.stderr}")
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Audio metadata extraction failed: {str(e)}"
+        logging.error(f"Audio metadata error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "audio_metadata_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def extract_text_ocr(
+    image_path: str,
+    language: str = "eng"
+) -> Union[str, TextContent]:
+    """
+    Extract text from image using OCR.
+    
+    Args:
+        image_path: Path to image file
+        language: OCR language (eng, chi_sim, etc.)
+        
+    Returns:
+        TextContent with extracted text
+    """
+    try:
+        path = validate_file_path(image_path)
+        
+        logging.info(f"🔍 OCR extracting from image: {path}")
+        
+        try:
+            import pytesseract
+            
+            img = Image.open(path)
+            text = pytesseract.image_to_string(img, lang=language)
+            
+            result = {
+                "file_name": path.name,
+                "image_size": img.size,
+                "extracted_text": text,
+                "text_length": len(text),
+                "word_count": len(text.split()),
+                "language": language,
+                "method": "pytesseract"
+            }
+            
+            logging.info(f"✅ OCR extracted: {len(text)} chars")
+            
+            action_response = ActionResponse(
+                success=True,
+                message=result,
+                metadata={"file_path": str(path)}
+            )
+            
+        except ImportError:
+            # Fallback to a simpler method or error
+            raise ImportError("pytesseract not installed. Install with: pip install pytesseract")
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"OCR extraction failed: {str(e)}"
+        logging.error(f"OCR error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "ocr_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def analyze_image_ai(
+    image_path: str,
+    prompt: str = "Describe this image in detail"
+) -> Union[str, TextContent]:
+    """
+    Analyze image using AI (OpenAI Vision API).
+    
+    Args:
+        image_path: Path to image file
+        prompt: Prompt for AI analysis
+        
+    Returns:
+        TextContent with AI analysis
+    """
+    try:
+        path = validate_file_path(image_path)
+
+        logging.info(f"🤖 AI analyzing image: {path}")
+
+        client, model = _make_vision_client()
+
+        # Encode image
+        with open(path, "rb") as img_file:
+            img_base64 = base64.b64encode(img_file.read()).decode('utf-8')
+
+        # Call Vision API
+        started = time.perf_counter()
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": prompt},
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": f"data:image/jpeg;base64,{img_base64}"
+                            }
+                        }
+                    ]
+                }
+            ],
+            max_tokens=500
+        )
+        latency_seconds = round(time.perf_counter() - started, 3)
+
+        analysis = response.choices[0].message.content
+
+        usage = getattr(response, "usage", None)
+        result = {
+            "file_name": path.name,
+            "prompt": prompt,
+            "analysis": analysis,
+            "model": model,
+            "provider_receipt": {
+                "provider": os.getenv("PERCEPTION_VISION_PROVIDER", "auto"),
+                "response_id": getattr(response, "id", None),
+                "response_model": getattr(response, "model", model),
+                "finish_reason": getattr(response.choices[0], "finish_reason", None),
+                "usage": {
+                    "prompt_tokens": getattr(usage, "prompt_tokens", None),
+                    "completion_tokens": getattr(usage, "completion_tokens", None),
+                    "total_tokens": getattr(usage, "total_tokens", None),
+                },
+                "latency_seconds": latency_seconds,
+            },
+        }
+        
+        logging.info(f"✅ AI analysis completed")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"file_path": str(path)}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"AI image analysis failed: {str(e)}"
+        logging.error(f"AI analysis error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "ai_analysis_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def extract_video_keyframes(
+    video_path: str,
+    num_frames: int = 10
+) -> Union[str, TextContent]:
+    """
+    Extract keyframes from video.
+    
+    Args:
+        video_path: Path to video file
+        num_frames: Number of keyframes to extract
+        
+    Returns:
+        TextContent with keyframe information
+    """
+    try:
+        path = validate_file_path(video_path)
+        
+        num_frames = max(1, num_frames)
+        
+        logging.info(f"🎬 Extracting keyframes from video: {path}")
+        
+        video = cv2.VideoCapture(str(path))
+        
+        fps = video.get(cv2.CAP_PROP_FPS)
+        frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
+        duration = frame_count / fps if fps > 0 else 0
+        
+        # Calculate frame interval
+        interval = max(1, frame_count // num_frames)
+        
+        keyframes = []
+        frame_num = 0
+        
+        while len(keyframes) < num_frames and video.isOpened():
+            ret, frame = video.read()
+            if not ret:
+                break
+            
+            if frame_num % interval == 0:
+                timestamp = frame_num / fps if fps > 0 else 0
+                keyframes.append({
+                    "frame_number": frame_num,
+                    "timestamp": round(timestamp, 2),
+                    "shape": frame.shape if frame is not None else None
+                })
+            
+            frame_num += 1
+        
+        video.release()
+        
+        result = {
+            "file_name": path.name,
+            "duration": duration,
+            "total_frames": frame_count,
+            "fps": fps,
+            "keyframes_extracted": len(keyframes),
+            "keyframes": keyframes
+        }
+        
+        logging.info(f"✅ Extracted {len(keyframes)} keyframes")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"file_path": str(path)}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Keyframe extraction failed: {str(e)}"
+        logging.error(f"Video error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "video_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def analyze_video_ai(
+    video_path: str,
+    num_frames: int = 5,
+    prompt: str = "Analyze this video and describe what's happening"
+) -> Union[str, TextContent]:
+    """
+    Analyze video content using AI vision on keyframes.
+    
+    Args:
+        video_path: Path to video file
+        num_frames: Number of frames to analyze
+        prompt: Analysis prompt for AI
+        
+    Returns:
+        TextContent with AI analysis
+    """
+    video = None
+    try:
+        path = validate_file_path(video_path)
+
+        num_frames = max(1, num_frames)
+
+        logging.info(f"🤖 AI analyzing video: {path}")
+
+        client, model = _make_vision_client()
+
+        # Extract keyframes
+        video = cv2.VideoCapture(str(path))
+        fps = video.get(cv2.CAP_PROP_FPS)
+        frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
+        interval = max(1, frame_count // num_frames)
+        
+        # Extract and encode frames
+        frame_analyses = []
+        frame_num = 0
+        frames_analyzed = 0
+        
+        while frames_analyzed < num_frames and video.isOpened():
+            ret, frame = video.read()
+            if not ret:
+                break
+            
+            if frame_num % interval == 0:
+                # Encode frame
+                _, buffer = cv2.imencode('.jpg', frame)
+                img_base64 = base64.b64encode(buffer).decode('utf-8')
+                
+                # Analyze with GPT-4 Vision
+                started = time.perf_counter()
+                response = client.chat.completions.create(
+                    model=model,
+                    messages=[
+                        {
+                            "role": "user",
+                            "content": [
+                                {"type": "text", "text": f"{prompt} (Frame {frames_analyzed + 1}/{num_frames})"},
+                                {
+                                    "type": "image_url",
+                                    "image_url": {"url": f"data:image/jpeg;base64,{img_base64}"}
+                                }
+                            ]
+                        }
+                    ],
+                    max_tokens=300
+                )
+                latency_seconds = round(time.perf_counter() - started, 3)
+                
+                timestamp = frame_num / fps if fps > 0 else 0
+                analysis = response.choices[0].message.content
+                
+                usage = getattr(response, "usage", None)
+                frame_analyses.append({
+                    "frame_number": frame_num,
+                    "timestamp": round(timestamp, 2),
+                    "analysis": analysis,
+                    "provider_receipt": {
+                        "provider": os.getenv("PERCEPTION_VISION_PROVIDER", "auto"),
+                        "response_id": getattr(response, "id", None),
+                        "response_model": getattr(response, "model", model),
+                        "finish_reason": getattr(response.choices[0], "finish_reason", None),
+                        "usage": {
+                            "prompt_tokens": getattr(usage, "prompt_tokens", None),
+                            "completion_tokens": getattr(usage, "completion_tokens", None),
+                            "total_tokens": getattr(usage, "total_tokens", None),
+                        },
+                        "latency_seconds": latency_seconds,
+                    },
+                })
+                
+                frames_analyzed += 1
+            
+            frame_num += 1
+
+        # Generate overall summary
+        combined_analyses = "\n\n".join([f"Frame {i+1} (t={a['timestamp']}s): {a['analysis']}" 
+                                          for i, a in enumerate(frame_analyses)])
+        
+        result = {
+            "file_name": path.name,
+            "frames_analyzed": len(frame_analyses),
+            "analyses": frame_analyses,
+            "combined_analysis": combined_analyses
+        }
+        
+        logging.info(f"✅ Analyzed {len(frame_analyses)} frames")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"file_path": str(path)}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Video analysis failed: {str(e)}"
+        logging.error(f"Video analysis error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "video_analysis_error"}
+        )
+
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+    finally:
+        # Release the native decoder/file handle even when a per-frame Vision
+        # API call raises mid-loop (the most likely failure point), otherwise
+        # the VideoCapture leaks until GC finalizes it.
+        if video is not None:
+            video.release()
+
+
+async def trim_audio(
+    audio_path: str,
+    start_time: float,
+    duration: float | None = None,
+    output_path: str | None = None
+) -> Union[str, TextContent]:
+    """
+    Trim audio file to specified time range using ffmpeg.
+    
+    Args:
+        audio_path: Path to audio file
+        start_time: Start time in seconds
+        duration: Duration in seconds (None for trim to end)
+        output_path: Output file path (None for auto-generate)
+        
+    Returns:
+        TextContent with trimmed audio info
+    """
+    try:
+        path = validate_file_path(audio_path)
+        
+        logging.info(f"✂️ Trimming audio: {path}")
+        
+        # Generate output path if not provided
+        if output_path is None:
+            output_path = str(path.parent / f"{path.stem}_trimmed{path.suffix}")
+        
+        # Build ffmpeg command
+        cmd = ["ffmpeg", "-i", str(path), "-ss", str(start_time)]
+        
+        if duration is not None:
+            cmd.extend(["-t", str(duration)])
+        
+        cmd.extend(["-c", "copy", "-y", output_path])
+        
+        # Execute ffmpeg
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
+        
+        if result.returncode != 0:
+            raise RuntimeError(f"ffmpeg failed: {result.stderr}")
+        
+        output_file = Path(output_path)
+        
+        response_data = {
+            "input_file": str(path),
+            "output_file": str(output_file),
+            "start_time": start_time,
+            "duration": duration,
+            "file_size": output_file.stat().st_size if output_file.exists() else 0
+        }
+        
+        logging.info(f"✅ Trimmed audio saved to: {output_file}")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=response_data,
+            metadata={"output_path": str(output_file)}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Audio trim failed: {str(e)}"
+        logging.error(f"Audio trim error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "audio_trim_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_image_metadata(
+    image_path: str
+) -> Union[str, TextContent]:
+    """
+    Get detailed image metadata including EXIF data.
+    
+    Args:
+        image_path: Path to image file
+        
+    Returns:
+        TextContent with image metadata
+    """
+    try:
+        path = validate_file_path(image_path)
+        
+        logging.info(f"📷 Getting image metadata: {path}")
+        
+        img = Image.open(path)
+        
+        # Basic metadata
+        metadata = {
+            "file_name": path.name,
+            "format": img.format,
+            "mode": img.mode,
+            "size": img.size,
+            "width": img.width,
+            "height": img.height,
+            "file_size": path.stat().st_size
+        }
+        
+        # Try to get EXIF data
+        try:
+            from PIL.ExifTags import TAGS
+            exif_data = {}
+            
+            if hasattr(img, '_getexif') and img._getexif():
+                exif = img._getexif()
+                for tag_id, value in exif.items():
+                    tag = TAGS.get(tag_id, tag_id)
+                    exif_data[tag] = str(value)
+            
+            if exif_data:
+                metadata["exif"] = exif_data
+        except Exception as e:
+            logging.debug(f"No EXIF data: {e}")
+        
+        # Image info
+        if hasattr(img, 'info'):
+            # PIL's img.info routinely carries non-JSON-serializable values --
+            # most notably the raw ICC color profile / EXIF blob as `bytes`
+            # (present in almost every real photo, screenshot or design export).
+            # Copying them verbatim would make the final json.dumps() raise
+            # TypeError, turning a valid image into a failure response. Summarize
+            # bytes as a size marker so metadata extraction still succeeds.
+            metadata["info"] = {
+                k: (f"<{len(v)} bytes>" if isinstance(v, (bytes, bytearray)) else v)
+                for k, v in img.info.items()
+            }
+        
+        result = {
+            "metadata": metadata,
+            "has_exif": "exif" in metadata
+        }
+        
+        logging.info(f"✅ Image metadata extracted")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"file_path": str(path)}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Metadata extraction failed: {str(e)}"
+        logging.error(f"Metadata error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "metadata_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )

chapter4/perception-tools/src/multimodal_tools.py

@@ -0,0 +1,583 @@
+"""
+Multimodal understanding tools: web, documents, images, and videos.
+"""
+import json
+import logging
+import os
+import traceback
+from pathlib import Path
+from typing import Optional, Union
+import base64
+
+import requests
+from bs4 import BeautifulSoup
+from dotenv import load_dotenv
+from mcp.types import TextContent
+from pydantic import Field
+
+from base import ActionResponse, validate_file_path, download_file_from_url, is_url
+
+
+load_dotenv()
+
+
+async def read_webpage(
+    url: str,
+    extract_text: bool = True,
+    extract_links: bool = False
+) -> Union[str, TextContent]:
+    """
+    Read and extract content from a webpage.
+    
+    Args:
+        url: URL of the webpage
+        extract_text: Whether to extract main text content
+        extract_links: Whether to extract all links
+        
+    Returns:
+        TextContent with extracted webpage content
+    """
+    try:
+        logging.info(f"📄 Reading webpage: {url}")
+        
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+        }
+        
+        response = requests.get(url, headers=headers, timeout=30)
+        response.raise_for_status()
+        
+        soup = BeautifulSoup(response.content, 'html.parser')
+        
+        result = {
+            "url": url,
+            "title": soup.title.string if soup.title else "No title"
+        }
+        
+        if extract_text:
+            # Remove script and style elements
+            for script in soup(["script", "style"]):
+                script.decompose()
+            
+            text = soup.get_text()
+            lines = (line.strip() for line in text.splitlines())
+            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+            text = ' '.join(chunk for chunk in chunks if chunk)
+            
+            result["text"] = text[:5000]  # Limit to first 5000 chars
+            result["text_length"] = len(text)
+        
+        if extract_links:
+            links = []
+            for link in soup.find_all('a', href=True):
+                links.append({
+                    "text": link.get_text().strip(),
+                    "href": link['href']
+                })
+            result["links"] = links[:50]  # Limit to first 50 links
+        
+        logging.info(f"✅ Successfully extracted webpage content")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"url": url}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Webpage reading failed: {str(e)}"
+        logging.error(f"Webpage error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "webpage_error", "url": url}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+def _sniff_document_type(path: Path) -> Optional[str]:
+    """Detect .pdf/.docx/.pptx from magic bytes when the extension is unusable.
+
+    A URL's path often carries no real extension (e.g.
+    https://arxiv.org/pdf/2301.07041 -> ".07041"), so the downloaded temp
+    file's suffix cannot be trusted to identify the format.
+    """
+    try:
+        with open(path, 'rb') as f:
+            header = f.read(4)
+        if header.startswith(b'%PDF'):
+            return '.pdf'
+        if header.startswith(b'PK\x03\x04'):
+            import zipfile
+            with zipfile.ZipFile(path) as zf:
+                names = zf.namelist()
+            if any(n.startswith('word/') for n in names):
+                return '.docx'
+            if any(n.startswith('ppt/') for n in names):
+                return '.pptx'
+    except Exception as e:
+        logging.debug(f"Document type sniffing failed: {e}")
+    return None
+
+
+async def read_document(
+    file_path: str,
+    extract_images: bool = False
+) -> Union[str, TextContent]:
+    """
+    Read and extract content from documents (PDF, DOCX, PPTX).
+    
+    Args:
+        file_path: Path to the document file (or URL)
+        extract_images: Whether to extract images from document
+        
+    Returns:
+        TextContent with extracted document content
+    """
+    try:
+        # Handle URL downloads
+        if is_url(file_path):
+            logging.info(f"📥 Downloading document from URL")
+            temp_path, _ = download_file_from_url(file_path)
+            file_path = temp_path
+        
+        path = validate_file_path(file_path)
+        
+        logging.info(f"📄 Reading document: {path}")
+        
+        file_ext = path.suffix.lower()
+        if file_ext not in ('.pdf', '.docx', '.pptx'):
+            # The suffix came from the URL path and may be meaningless
+            # (".07041", ".tmp"), so fall back to the file's magic bytes.
+            file_ext = _sniff_document_type(path) or file_ext
+        
+        # PDF extraction
+        if file_ext == '.pdf':
+            import PyPDF2
+            
+            with open(path, 'rb') as file:
+                reader = PyPDF2.PdfReader(file)
+                text = ""
+                for page in reader.pages:
+                    text += page.extract_text() + "\n"
+                
+                result = {
+                    "file_name": path.name,
+                    "file_type": "pdf",
+                    "page_count": len(reader.pages),
+                    "text": text[:10000],  # Limit size
+                    "text_length": len(text)
+                }
+        
+        # DOCX extraction
+        elif file_ext == '.docx':
+            from docx import Document
+            
+            doc = Document(path)
+            text = "\n".join([para.text for para in doc.paragraphs])
+            
+            result = {
+                "file_name": path.name,
+                "file_type": "docx",
+                "paragraph_count": len(doc.paragraphs),
+                "text": text[:10000],
+                "text_length": len(text)
+            }
+        
+        # PPTX extraction
+        elif file_ext == '.pptx':
+            from pptx import Presentation
+            
+            prs = Presentation(path)
+            text = ""
+            for slide in prs.slides:
+                for shape in slide.shapes:
+                    if hasattr(shape, "text"):
+                        text += shape.text + "\n"
+            
+            result = {
+                "file_name": path.name,
+                "file_type": "pptx",
+                "slide_count": len(prs.slides),
+                "text": text[:10000],
+                "text_length": len(text)
+            }
+        
+        else:
+            raise ValueError(f"Unsupported file type: {file_ext}")
+        
+        logging.info(f"✅ Successfully extracted document content")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"file_path": str(path), "file_type": file_ext}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Document reading failed: {str(e)}"
+        logging.error(f"Document error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "document_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def parse_image(
+    image_path: str,
+    use_llm: bool = True
+) -> Union[str, TextContent]:
+    """
+    Parse and understand image content.
+    
+    Args:
+        image_path: Path to image file or URL
+        use_llm: Whether to use LLM for image understanding
+        
+    Returns:
+        TextContent with image analysis
+    """
+    try:
+        # Handle URL downloads
+        if is_url(image_path):
+            logging.info(f"📥 Downloading image from URL")
+            temp_path, _ = download_file_from_url(image_path)
+            image_path = temp_path
+        
+        path = validate_file_path(image_path)
+        
+        logging.info(f"🖼️ Parsing image: {path}")
+        
+        from PIL import Image
+        
+        img = Image.open(path)
+        
+        result = {
+            "file_name": path.name,
+            "format": img.format,
+            "mode": img.mode,
+            "size": img.size,
+            "width": img.width,
+            "height": img.height
+        }
+        
+        # If LLM analysis requested, encode image for vision API
+        if use_llm:
+            with open(path, 'rb') as img_file:
+                img_base64 = base64.b64encode(img_file.read()).decode('utf-8')
+                result["base64_data"] = img_base64[:100] + "..."  # Truncated for display
+                result["note"] = "Full base64 data available for vision API analysis"
+        
+        logging.info(f"✅ Successfully parsed image")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"file_path": str(path)}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Image parsing failed: {str(e)}"
+        logging.error(f"Image error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "image_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def parse_video(
+    video_path: str,
+    extract_frames: bool = False,
+    frame_interval: int = 30
+) -> Union[str, TextContent]:
+    """
+    Parse and extract information from video files.
+    
+    Args:
+        video_path: Path to video file or URL
+        extract_frames: Whether to extract sample frames
+        frame_interval: Extract one frame every N seconds
+        
+    Returns:
+        TextContent with video metadata
+    """
+    try:
+        # Handle URL downloads
+        if is_url(video_path):
+            logging.info(f"📥 Downloading video from URL")
+            temp_path, _ = download_file_from_url(video_path, max_size_mb=500)
+            video_path = temp_path
+        
+        path = validate_file_path(video_path)
+        
+        logging.info(f"🎥 Parsing video: {path}")
+        
+        import cv2
+        
+        video = cv2.VideoCapture(str(path))
+        
+        fps = video.get(cv2.CAP_PROP_FPS)
+        frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
+        width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        duration = frame_count / fps if fps > 0 else 0
+        
+        result = {
+            "file_name": path.name,
+            "duration_seconds": duration,
+            "fps": fps,
+            "frame_count": frame_count,
+            "resolution": f"{width}x{height}",
+            "width": width,
+            "height": height
+        }
+        
+        video.release()
+        
+        logging.info(f"✅ Successfully parsed video metadata")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"file_path": str(path)}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Video parsing failed: {str(e)}"
+        logging.error(f"Video error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "video_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def download_youtube_video(
+    url: str,
+    output_dir: str = ".",
+    max_resolution: str = "720p"
+) -> Union[str, TextContent]:
+    """
+    Download YouTube video using yt-dlp.
+    
+    Args:
+        url: YouTube video URL
+        output_dir: Directory to save video
+        max_resolution: Maximum resolution (360p, 480p, 720p, 1080p)
+        
+    Returns:
+        TextContent with download result
+    """
+    try:
+        logging.info(f"📥 Downloading YouTube video: {url}")
+        
+        try:
+            import yt_dlp
+            
+            output_template = Path(output_dir) / '%(title)s.%(ext)s'
+            
+            ydl_opts = {
+                'format': f'best[height<={max_resolution[:-1]}]',
+                'outtmpl': str(output_template),
+                'quiet': False
+            }
+            
+            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                info = ydl.extract_info(url, download=True)
+                
+                result = {
+                    "title": info['title'],
+                    "duration": info.get('duration'),
+                    "output_dir": output_dir,
+                    "resolution": max_resolution,
+                    "video_id": info['id']
+                }
+                
+                logging.info(f"✅ Downloaded: {info['title']}")
+                
+                action_response = ActionResponse(
+                    success=True,
+                    message=result,
+                    metadata={"url": url}
+                )
+                
+        except ImportError:
+            raise ImportError("yt-dlp not installed. Install with: pip install yt-dlp")
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"YouTube download failed: {str(e)}"
+        logging.error(f"YouTube download error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "youtube_download_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def extract_youtube_transcript(
+    video_id: str,
+    language_code: str = "en",
+    translate_to_language: str | None = None
+) -> Union[str, TextContent]:
+    """
+    Extract transcript from a YouTube video.
+    
+    Args:
+        video_id: YouTube video ID or URL
+        language_code: Language code for the transcript (default: en)
+        translate_to_language: Translate transcript to this language if provided
+        
+    Returns:
+        TextContent with transcript data
+    """
+    try:
+        from youtube_transcript_api import YouTubeTranscriptApi
+        
+        # Clean video_id if full URL was provided
+        if "youtube.com" in video_id or "youtu.be" in video_id:
+            if "?v=" in video_id:
+                video_id = video_id.split("?v=")[-1].split("&")[0]
+            elif "youtu.be/" in video_id:
+                video_id = video_id.split("youtu.be/")[-1].split("?")[0]
+        
+        logging.info(f"📺 Extracting transcript for video ID: {video_id}")
+        
+        # Get transcript using correct API
+        if translate_to_language:
+            transcript_list = YouTubeTranscriptApi().list(video_id)
+            try:
+                transcript = transcript_list.find_transcript([language_code])
+            except Exception:
+                # If specified language not found, get any available transcript
+                transcript = transcript_list.find_generated_transcript(["en"])
+            # Translate to target language
+            fetched_transcript = transcript.translate(translate_to_language).fetch()
+            transcript_data = fetched_transcript.snippets
+        else:
+            try:
+                # Use fetch method which returns FetchedTranscript
+                fetched_transcript = YouTubeTranscriptApi().fetch(
+                    video_id, 
+                    languages=(language_code,)
+                )
+                transcript_data = fetched_transcript.snippets
+            except Exception:
+                # Fallback to English
+                fetched_transcript = YouTubeTranscriptApi().fetch(video_id, languages=("en",))
+                transcript_data = fetched_transcript.snippets
+        
+        # Format transcript
+        formatted_transcript = []
+        for entry in transcript_data:
+            # Access as object attributes, not dictionary
+            start_time = entry.start if hasattr(entry, 'start') else entry.get('start', 0)
+            text = entry.text if hasattr(entry, 'text') else entry.get('text', '')
+            minutes, seconds = divmod(int(start_time), 60)
+            timestamp = f"{minutes:02d}:{seconds:02d}"
+            formatted_transcript.append({
+                "timestamp": timestamp,
+                "text": text
+            })
+        
+        # Create full text version
+        full_text = " ".join([
+            entry.text if hasattr(entry, 'text') else entry.get('text', '') 
+            for entry in transcript_data
+        ])
+        
+        result = {
+            "video_id": video_id,
+            "language": translate_to_language if translate_to_language else language_code,
+            "transcript": formatted_transcript[:100],  # Limit to first 100 entries
+            "total_entries": len(transcript_data),
+            "full_text": full_text[:5000],  # Limit full text to 5000 chars
+            "full_text_length": len(full_text)
+        }
+        
+        logging.info(f"✅ Successfully extracted transcript ({len(transcript_data)} entries)")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={
+                "video_id": video_id,
+                "language": language_code,
+                "translated": translate_to_language is not None
+            }
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"YouTube transcript extraction failed: {str(e)}"
+        logging.error(f"YouTube error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "youtube_error", "video_id": video_id}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )

chapter4/perception-tools/src/private_data_tools.py

@@ -0,0 +1,270 @@
+"""
+Private data source tools: Google Calendar, Notion.
+"""
+import json
+import logging
+import os
+import traceback
+from datetime import datetime, timedelta
+from typing import Union
+
+from dotenv import load_dotenv
+from mcp.types import TextContent
+
+from base import ActionResponse
+
+
+load_dotenv()
+
+
+async def get_calendar_events(
+    start_date: str | None = None,
+    end_date: str | None = None,
+    calendar_id: str = "primary",
+    max_results: int = 10
+) -> Union[str, TextContent]:
+    """
+    Get events from Google Calendar.
+    
+    Args:
+        start_date: Start date (ISO format, defaults to today)
+        end_date: End date (ISO format, defaults to 7 days from now)
+        calendar_id: Calendar ID (default: primary)
+        max_results: Maximum number of events to return
+        
+    Returns:
+        TextContent with calendar events
+    """
+    try:
+        from google.oauth2.credentials import Credentials
+        from googleapiclient.discovery import build
+        from google.auth.transport.requests import Request
+        import pickle
+        
+        logging.info(f"📅 Getting calendar events")
+        
+        # Token file path
+        token_path = os.path.expanduser("~/.perception-tools/google_token.pickle")
+        
+        if not os.path.exists(token_path):
+            action_response = ActionResponse(
+                success=False,
+                message="Google Calendar not configured. Please run setup to authenticate.",
+                metadata={"error_type": "missing_credentials"}
+            )
+            return TextContent(
+                type="text",
+                text=json.dumps(action_response.model_dump())
+            )
+        
+        # Load credentials
+        with open(token_path, 'rb') as token:
+            creds = pickle.load(token)
+        
+        # Refresh if expired
+        if creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        
+        service = build('calendar', 'v3', credentials=creds)
+        
+        # Set default date range if not provided
+        if not start_date:
+            start_date = datetime.utcnow().isoformat() + 'Z'
+        if not end_date:
+            end_dt = datetime.utcnow() + timedelta(days=7)
+            end_date = end_dt.isoformat() + 'Z'
+        
+        # Query calendar
+        events_result = service.events().list(
+            calendarId=calendar_id,
+            timeMin=start_date,
+            timeMax=end_date,
+            maxResults=max_results,
+            singleEvents=True,
+            orderBy='startTime'
+        ).execute()
+        
+        events = events_result.get('items', [])
+        
+        formatted_events = []
+        for event in events:
+            start = event['start'].get('dateTime', event['start'].get('date'))
+            end = event['end'].get('dateTime', event['end'].get('date'))
+            
+            formatted_events.append({
+                "id": event['id'],
+                "summary": event.get('summary', 'No title'),
+                "start": start,
+                "end": end,
+                "location": event.get('location'),
+                "description": event.get('description'),
+                "attendees": [a.get('email') for a in event.get('attendees', [])]
+            })
+        
+        logging.info(f"✅ Found {len(formatted_events)} calendar events")
+        
+        action_response = ActionResponse(
+            success=True,
+            message={
+                "events": formatted_events,
+                "count": len(formatted_events),
+                "calendar_id": calendar_id
+            },
+            metadata={"start_date": start_date, "end_date": end_date}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except ImportError:
+        error_msg = "Google Calendar libraries not installed. Install with: pip install google-auth-oauthlib google-auth-httplib2 google-api-python-client"
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "missing_library"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Calendar query failed: {str(e)}"
+        logging.error(f"Calendar error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "calendar_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def search_notion(
+    query: str,
+    database_id: str | None = None,
+    page_size: int = 10
+) -> Union[str, TextContent]:
+    """
+    Search Notion workspace or specific database.
+    
+    Args:
+        query: Search query
+        database_id: Optional specific database ID
+        page_size: Number of results per page
+        
+    Returns:
+        TextContent with Notion search results
+    """
+    try:
+        from notion_client import Client
+        
+        logging.info(f"📝 Searching Notion for: {query}")
+        
+        api_key = os.getenv("NOTION_API_KEY")
+        
+        if not api_key:
+            action_response = ActionResponse(
+                success=False,
+                message="Notion API key not configured. Set NOTION_API_KEY.",
+                metadata={"error_type": "missing_credentials"}
+            )
+            return TextContent(
+                type="text",
+                text=json.dumps(action_response.model_dump())
+            )
+        
+        notion = Client(auth=api_key)
+        
+        if database_id:
+            # Search specific database
+            response = notion.databases.query(
+                database_id=database_id,
+                filter={
+                    "property": "Name",
+                    "rich_text": {
+                        "contains": query
+                    }
+                },
+                page_size=page_size
+            )
+        else:
+            # Search entire workspace
+            response = notion.search(
+                query=query,
+                page_size=page_size
+            )
+        
+        results = []
+        for item in response.get("results", []):
+            result_data = {
+                "id": item["id"],
+                "type": item["object"],
+                "url": item.get("url"),
+                "created_time": item.get("created_time"),
+                "last_edited_time": item.get("last_edited_time")
+            }
+            
+            # Extract title/name
+            if "properties" in item:
+                for prop_name, prop_value in item["properties"].items():
+                    if prop_value.get("type") == "title" and prop_value.get("title"):
+                        title_parts = [t.get("plain_text", "") for t in prop_value["title"]]
+                        result_data["title"] = "".join(title_parts)
+                        break
+            
+            results.append(result_data)
+        
+        logging.info(f"✅ Found {len(results)} Notion items")
+        
+        action_response = ActionResponse(
+            success=True,
+            message={
+                "query": query,
+                "results": results,
+                "count": len(results)
+            },
+            metadata={"database_id": database_id}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except ImportError:
+        error_msg = "Notion SDK not installed. Install with: pip install notion-client"
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "missing_library"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Notion search failed: {str(e)}"
+        logging.error(f"Notion error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "notion_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )

chapter4/perception-tools/src/pubchem_tools.py

@@ -0,0 +1,544 @@
+"""
+PubChem chemical compound data tools.
+Based on AWorld MCP server implementation.
+"""
+import json
+import logging
+import time
+import traceback
+from typing import Union, Literal
+from urllib.parse import quote, urlsplit, urlunsplit
+
+import requests
+from dotenv import load_dotenv
+from mcp.types import TextContent
+from pydantic import BaseModel, Field
+
+from base import ActionResponse
+
+
+load_dotenv()
+
+
+class CompoundData(BaseModel):
+    """Structured compound data from PubChem."""
+    
+    cid: int | None = None
+    name: str | None = None
+    molecular_formula: str | None = None
+    molecular_weight: float | None = None
+    smiles: str | None = None
+    inchi: str | None = None
+    synonyms: list[str] = []
+
+
+class PubChemMetadata(BaseModel):
+    """Metadata for PubChem operation results."""
+    
+    query_type: str
+    query_value: str
+    api_endpoint: str
+    response_time: float
+    total_results: int | None = None
+    rate_limit_delay: float | None = None
+    error_type: str | None = None
+
+
+class PubChemClient:
+    """PubChem API client with rate limiting."""
+    
+    def __init__(self):
+        self.base_url = "https://pubchem.ncbi.nlm.nih.gov/rest/pug"
+        self.request_delay = 0.2  # 200ms delay to stay under 5 req/sec limit
+        self.last_request_time = 0.0
+        self.timeout = 30
+        
+        self.session = requests.Session()
+        self.session.headers.update({
+            "User-Agent": "PerceptionToolsMCP/1.0",
+            "Accept": "application/json"
+        })
+    
+    def _rate_limit(self) -> float:
+        """Enforce rate limiting to comply with PubChem usage policy."""
+        current_time = time.time()
+        time_since_last = current_time - self.last_request_time
+        
+        if time_since_last < self.request_delay:
+            delay = self.request_delay - time_since_last
+            time.sleep(delay)
+            self.last_request_time = time.time()
+            return delay
+        
+        self.last_request_time = current_time
+        return 0.0
+    
+    @staticmethod
+    def _listkey_poll_url(url: str, list_key: str) -> str:
+        """Convert an asynchronous PUG REST request into its ListKey poll URL."""
+        parsed = urlsplit(url)
+        compound_marker = "/compound/"
+        compound_index = parsed.path.find(compound_marker)
+        if compound_index < 0:
+            raise requests.RequestException("PubChem ListKey response for an unsupported endpoint")
+
+        operation_start = compound_index + len(compound_marker)
+        operation_indexes = [
+            index
+            for marker in ("/property/", "/cids/", "/synonyms/")
+            if (index := parsed.path.find(marker, operation_start)) >= 0
+        ]
+        if not operation_indexes:
+            raise requests.RequestException("PubChem ListKey response did not identify a poll operation")
+
+        operation_index = min(operation_indexes)
+        poll_path = (
+            parsed.path[:operation_start]
+            + f"listkey/{quote(list_key, safe='')}"
+            + parsed.path[operation_index:]
+        )
+        return urlunsplit((parsed.scheme, parsed.netloc, poll_path, parsed.query, parsed.fragment))
+
+    def make_request(
+        self,
+        url: str,
+        params: dict = None,
+        max_retries: int = 12,
+        _origin_url: str | None = None,
+        _started_at: float | None = None,
+    ) -> tuple[dict | None, float]:
+        """Make a rate-limited request to PubChem API with retry for async operations."""
+        origin_url = _origin_url or url
+        started_at = _started_at if _started_at is not None else time.perf_counter()
+        self._rate_limit()
+        
+        try:
+            response = self.session.get(url, params=params, timeout=self.timeout)
+            
+            if response.status_code == 200:
+                return response.json(), time.perf_counter() - started_at
+            elif response.status_code == 202:
+                # PUG REST returns a ListKey for long-running searches. Polling the
+                # original URL starts a new job, so switch to the documented
+                # list-key endpoint and retain the complete request latency.
+                if max_retries > 0:
+                    waiting = response.json().get("Waiting", {})
+                    list_key = str(waiting.get("ListKey", "")).strip()
+                    if not list_key:
+                        raise requests.RequestException("PubChem async response omitted ListKey")
+                    poll_url = self._listkey_poll_url(url, list_key)
+                    logging.info("PubChem async operation, waiting 2s before polling ListKey")
+                    time.sleep(2)
+                    return self.make_request(
+                        poll_url,
+                        params,
+                        max_retries - 1,
+                        _origin_url=origin_url,
+                        _started_at=started_at,
+                    )
+                else:
+                    raise requests.RequestException("PubChem async operation timeout after retries")
+            elif response.status_code in {429, 500, 502, 503, 504} and max_retries > 0:
+                # A ListKey job can fail independently inside PubChem. Start a
+                # fresh copy of the original query in that case; otherwise
+                # retry the same endpoint. All retries remain bounded.
+                retry_url = origin_url if "/listkey/" in url else url
+                logging.warning(
+                    "PubChem transient HTTP %s; retrying %s",
+                    response.status_code,
+                    "original query" if retry_url == origin_url else "request",
+                )
+                time.sleep(2)
+                return self.make_request(
+                    retry_url,
+                    params,
+                    max_retries - 1,
+                    _origin_url=origin_url,
+                    _started_at=started_at,
+                )
+            else:
+                raise requests.RequestException(f"HTTP {response.status_code}: {response.text}")
+        
+        except requests.Timeout:
+            raise requests.RequestException(f"Request timeout after {self.timeout}s")
+        except requests.RequestException:
+            raise
+
+
+# Global client instance
+_client = None
+
+
+def get_client() -> PubChemClient:
+    """Get or create the global PubChem client."""
+    global _client
+    if _client is None:
+        _client = PubChemClient()
+    return _client
+
+
+async def search_compounds(
+    query: str,
+    search_type: Literal["name", "cid", "smiles", "inchi", "formula"] = "name",
+    max_results: int = 10
+) -> Union[str, TextContent]:
+    """
+    Search for chemical compounds in PubChem database.
+    
+    Args:
+        query: Search term or identifier
+        search_type: Type of search (name, cid, smiles, inchi, formula)
+        max_results: Maximum number of results (1-100)
+        
+    Returns:
+        TextContent with compound search results
+    """
+    try:
+        if not query or not query.strip():
+            raise ValueError("Search query is required")
+        
+        max_results = max(1, min(max_results, 100))
+        
+        logging.info(f"🔬 Searching PubChem for: {query} (type: {search_type})")
+        
+        client = get_client()
+        
+        # Build API URL based on search type
+        if search_type == "cid":
+            url = f"{client.base_url}/compound/cid/{quote(str(query))}/property/Title,MolecularFormula,MolecularWeight,CanonicalSMILES,InChI/JSON"
+        elif search_type == "name":
+            url = f"{client.base_url}/compound/name/{quote(query)}/property/Title,MolecularFormula,MolecularWeight,CanonicalSMILES,InChI/JSON"
+        elif search_type == "smiles":
+            url = f"{client.base_url}/compound/smiles/{quote(query)}/property/Title,MolecularFormula,MolecularWeight,CanonicalSMILES,InChI/JSON"
+        elif search_type == "inchi":
+            url = f"{client.base_url}/compound/inchi/{quote(query)}/property/Title,MolecularFormula,MolecularWeight,CanonicalSMILES,InChI/JSON"
+        elif search_type == "formula":
+            url = f"{client.base_url}/compound/formula/{quote(query)}/property/Title,MolecularFormula,MolecularWeight,CanonicalSMILES,InChI/JSON"
+        else:
+            raise ValueError(f"Unsupported search type: {search_type}")
+        
+        # Make API request
+        data, response_time = client.make_request(url)
+        
+        # Parse results
+        compounds = []
+        if data and "PropertyTable" in data and "Properties" in data["PropertyTable"]:
+            properties_list = data["PropertyTable"]["Properties"][:max_results]
+            
+            for prop in properties_list:
+                compound = CompoundData(
+                    cid=prop.get("CID"),
+                    name=prop.get("Title"),
+                    molecular_formula=prop.get("MolecularFormula"),
+                    molecular_weight=prop.get("MolecularWeight"),
+                    smiles=prop.get("CanonicalSMILES"),
+                    inchi=prop.get("InChI")
+                )
+                compounds.append(compound)
+        
+        # Format results
+        result = {
+            "query": query,
+            "search_type": search_type,
+            "compounds": [c.model_dump() for c in compounds],
+            "count": len(compounds)
+        }
+        
+        metadata = PubChemMetadata(
+            query_type=search_type,
+            query_value=query,
+            api_endpoint=url,
+            response_time=response_time,
+            total_results=len(compounds)
+        )
+        
+        logging.info(f"✅ Found {len(compounds)} compounds ({response_time:.2f}s)")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except ValueError as e:
+        error_msg = f"Invalid input: {str(e)}"
+        logging.error(f"PubChem search error: {error_msg}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "invalid_input"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Search failed: {str(e)}"
+        logging.error(f"PubChem error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "api_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_compound_properties(
+    cid: int,
+    properties: list[str] | None = None
+) -> Union[str, TextContent]:
+    """
+    Retrieve detailed chemical properties for a PubChem compound.
+    
+    Args:
+        cid: PubChem Compound ID
+        properties: List of property names (e.g., MolecularWeight, XLogP)
+        
+    Returns:
+        TextContent with compound properties
+    """
+    try:
+        if not cid or cid <= 0:
+            raise ValueError("Valid PubChem CID is required")
+        
+        if not properties:
+            properties = [
+                "MolecularWeight", "MolecularFormula", "CanonicalSMILES",
+                "InChI", "XLogP", "TPSA", "HBondDonorCount", "HBondAcceptorCount"
+            ]
+        
+        logging.info(f"🔬 Getting properties for CID: {cid}")
+        
+        client = get_client()
+        props_str = ",".join(properties)
+        url = f"{client.base_url}/compound/cid/{cid}/property/{props_str}/JSON"
+        
+        data, response_time = client.make_request(url)
+        
+        compound_props = {}
+        if data and "PropertyTable" in data and "Properties" in data["PropertyTable"]:
+            props_data = data["PropertyTable"]["Properties"][0]
+            compound_props = {k: v for k, v in props_data.items() if k != "CID"}
+        
+        result = {
+            "cid": cid,
+            "properties": compound_props
+        }
+        
+        metadata = PubChemMetadata(
+            query_type="properties",
+            query_value=str(cid),
+            api_endpoint=url,
+            response_time=response_time,
+            total_results=len(compound_props)
+        )
+        
+        logging.info(f"✅ Retrieved {len(compound_props)} properties")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Property retrieval failed: {str(e)}"
+        logging.error(f"PubChem error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "api_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_compound_synonyms(
+    cid: int,
+    max_synonyms: int = 20
+) -> Union[str, TextContent]:
+    """
+    Retrieve synonyms for a PubChem compound.
+    
+    Args:
+        cid: PubChem Compound ID
+        max_synonyms: Maximum number of synonyms (1-100)
+        
+    Returns:
+        TextContent with compound synonyms
+    """
+    try:
+        if not cid or cid <= 0:
+            raise ValueError("Valid PubChem CID is required")
+        
+        max_synonyms = max(1, min(max_synonyms, 100))
+        
+        logging.info(f"🔬 Getting synonyms for CID: {cid}")
+        
+        client = get_client()
+        url = f"{client.base_url}/compound/cid/{cid}/synonyms/JSON"
+        
+        data, response_time = client.make_request(url)
+        
+        synonyms = []
+        if data and "InformationList" in data and "Information" in data["InformationList"]:
+            info_list = data["InformationList"]["Information"]
+            if info_list and "Synonym" in info_list[0]:
+                synonyms = info_list[0]["Synonym"][:max_synonyms]
+        
+        result = {
+            "cid": cid,
+            "synonyms": synonyms,
+            "count": len(synonyms)
+        }
+        
+        metadata = PubChemMetadata(
+            query_type="synonyms",
+            query_value=str(cid),
+            api_endpoint=url,
+            response_time=response_time,
+            total_results=len(synonyms)
+        )
+        
+        logging.info(f"✅ Retrieved {len(synonyms)} synonyms")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Synonym retrieval failed: {str(e)}"
+        logging.error(f"PubChem error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "api_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def search_similar_compounds(
+    cid: int,
+    similarity_threshold: float = 0.9,
+    max_results: int = 10
+) -> Union[str, TextContent]:
+    """
+    Find structurally similar compounds.
+    
+    Args:
+        cid: Reference compound CID
+        similarity_threshold: Minimum similarity (0.0-1.0)
+        max_results: Maximum results (1-50)
+        
+    Returns:
+        TextContent with similar compounds
+    """
+    try:
+        if not cid or cid <= 0:
+            raise ValueError("Valid PubChem CID is required")
+        
+        similarity_threshold = max(0.0, min(similarity_threshold, 1.0))
+        max_results = max(1, min(max_results, 50))
+        
+        logging.info(f"🔬 Searching similar compounds to CID: {cid}")
+        
+        client = get_client()
+        threshold_percent = int(similarity_threshold * 100)
+        url = f"{client.base_url}/compound/fastsimilarity_2d/cid/{cid}/property/Title,MolecularFormula,MolecularWeight/JSON"
+        params = {
+            "Threshold": threshold_percent,
+            "MaxRecords": max_results
+        }
+        
+        data, response_time = client.make_request(url, params)
+        
+        similar_compounds = []
+        if data and "PropertyTable" in data and "Properties" in data["PropertyTable"]:
+            properties_list = data["PropertyTable"]["Properties"]
+            
+            for prop in properties_list:
+                if prop.get("CID") != cid:
+                    compound = CompoundData(
+                        cid=prop.get("CID"),
+                        name=prop.get("Title"),
+                        molecular_formula=prop.get("MolecularFormula"),
+                        molecular_weight=prop.get("MolecularWeight")
+                    )
+                    similar_compounds.append(compound)
+        
+        result = {
+            "reference_cid": cid,
+            "similarity_threshold": similarity_threshold,
+            "similar_compounds": [c.model_dump() for c in similar_compounds],
+            "count": len(similar_compounds)
+        }
+        
+        metadata = PubChemMetadata(
+            query_type="similarity",
+            query_value=str(cid),
+            api_endpoint=url,
+            response_time=response_time,
+            total_results=len(similar_compounds)
+        )
+        
+        logging.info(f"✅ Found {len(similar_compounds)} similar compounds")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Similarity search failed: {str(e)}"
+        logging.error(f"PubChem error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "api_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )

chapter4/perception-tools/src/public_data_tools.py

@@ -0,0 +1,862 @@
+"""
+Public data source tools: weather, stocks, currency, Wiki, ArXiv, Wayback Machine.
+"""
+import json
+import logging
+import os
+import time
+import traceback
+from datetime import datetime
+from typing import Union
+
+import requests
+from dotenv import load_dotenv
+from mcp.types import TextContent
+from pydantic import BaseModel, Field
+import wikipedia
+
+from base import ActionResponse
+
+
+load_dotenv()
+
+
+async def get_weather(
+    location: str,
+    latitude: float | None = None,
+    longitude: float | None = None
+) -> Union[str, TextContent]:
+    """
+    Get current weather information for a location using Open-Meteo API.
+    
+    Args:
+        location: City name for display purposes
+        latitude: Latitude coordinate (if not provided, will try to geocode location)
+        longitude: Longitude coordinate (if not provided, will try to geocode location)
+        
+    Returns:
+        TextContent with weather data
+    """
+    try:
+        logging.info(f"🌤️ Getting weather for: {location}")
+        
+        # If coordinates not provided, try to geocode the location
+        if latitude is None or longitude is None:
+            # Use Open-Meteo's geocoding API
+            geocode_url = "https://geocoding-api.open-meteo.com/v1/search"
+            geocode_params = {
+                "name": location,
+                "count": 1,
+                "language": "en",
+                "format": "json"
+            }
+            
+            geocode_response = requests.get(geocode_url, params=geocode_params, timeout=10)
+            geocode_response.raise_for_status()
+            geocode_data = geocode_response.json()
+            
+            if not geocode_data.get("results"):
+                raise ValueError(f"Location not found: {location}")
+            
+            first_result = geocode_data["results"][0]
+            latitude = first_result["latitude"]
+            longitude = first_result["longitude"]
+            location = first_result.get("name", location)
+            country = first_result.get("country", "")
+        else:
+            country = ""
+        
+        # Get weather data from Open-Meteo
+        weather_url = "https://api.open-meteo.com/v1/forecast"
+        weather_params = {
+            "latitude": latitude,
+            "longitude": longitude,
+            "current": "temperature_2m,relative_humidity_2m,apparent_temperature,precipitation,weather_code,wind_speed_10m,wind_direction_10m",
+            "timezone": "auto"
+        }
+        
+        response = requests.get(weather_url, params=weather_params, timeout=10)
+        response.raise_for_status()
+        
+        data = response.json()
+        current = data["current"]
+        
+        # Map weather codes to descriptions
+        # Based on WMO Weather interpretation codes
+        weather_codes = {
+            0: "Clear sky",
+            1: "Mainly clear", 2: "Partly cloudy", 3: "Overcast",
+            45: "Foggy", 48: "Depositing rime fog",
+            51: "Light drizzle", 53: "Moderate drizzle", 55: "Dense drizzle",
+            61: "Slight rain", 63: "Moderate rain", 65: "Heavy rain",
+            71: "Slight snow", 73: "Moderate snow", 75: "Heavy snow",
+            77: "Snow grains",
+            80: "Slight rain showers", 81: "Moderate rain showers", 82: "Violent rain showers",
+            85: "Slight snow showers", 86: "Heavy snow showers",
+            95: "Thunderstorm", 96: "Thunderstorm with slight hail", 99: "Thunderstorm with heavy hail"
+        }
+        
+        weather_code = current["weather_code"]
+        description = weather_codes.get(weather_code, "Unknown")
+        
+        result = {
+            "location": location,
+            "country": country,
+            "latitude": latitude,
+            "longitude": longitude,
+            "temperature": current["temperature_2m"],
+            "feels_like": current["apparent_temperature"],
+            "humidity": current["relative_humidity_2m"],
+            "precipitation": current["precipitation"],
+            "weather_code": weather_code,
+            "description": description,
+            "wind_speed": current["wind_speed_10m"],
+            "wind_direction": current["wind_direction_10m"],
+            "units": "metric",
+            "timestamp": current["time"],
+            "provider": "Open-Meteo"
+        }
+        
+        logging.info(f"✅ Weather: {result['temperature']}°C - {result['description']}")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"location": location, "provider": "Open-Meteo", "api_key_required": False}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Weather query failed: {str(e)}"
+        logging.error(f"Weather error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "weather_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_stock_price(
+    symbol: str,
+    interval: str = "1d"
+) -> Union[str, TextContent]:
+    """
+    Get stock price information.
+    
+    Args:
+        symbol: Stock ticker symbol (e.g., AAPL, TSLA)
+        interval: Data interval (1d, 1h, etc.)
+        
+    Returns:
+        TextContent with stock data
+    """
+    try:
+        logging.info(f"📈 Getting stock price for: {symbol}")
+        
+        # Using Yahoo Finance API (free, no key required)
+        url = f"https://query1.finance.yahoo.com/v8/finance/chart/{symbol}"
+        params = {
+            "interval": interval,
+            "range": "1d"
+        }
+        
+        headers = {
+            "User-Agent": "Mozilla/5.0"
+        }
+        
+        response = requests.get(url, params=params, headers=headers, timeout=10)
+        response.raise_for_status()
+        
+        data = response.json()
+        
+        if "chart" in data and "result" in data["chart"] and data["chart"]["result"]:
+            quote = data["chart"]["result"][0]["meta"]
+            
+            result = {
+                "symbol": symbol,
+                "currency": quote.get("currency", "USD"),
+                "current_price": quote.get("regularMarketPrice"),
+                "previous_close": quote.get("previousClose"),
+                "open": quote.get("regularMarketOpen"),
+                "day_high": quote.get("regularMarketDayHigh"),
+                "day_low": quote.get("regularMarketDayLow"),
+                "volume": quote.get("regularMarketVolume"),
+                "exchange": quote.get("exchangeName")
+            }
+            
+            logging.info(f"✅ Stock price: ${result['current_price']}")
+        else:
+            raise ValueError(f"Invalid response for symbol: {symbol}")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"symbol": symbol}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Stock query failed: {str(e)}"
+        logging.error(f"Stock error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "stock_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def convert_currency(
+    amount: float,
+    from_currency: str,
+    to_currency: str
+) -> Union[str, TextContent]:
+    """
+    Convert between currencies.
+    
+    Args:
+        amount: Amount to convert
+        from_currency: Source currency code (e.g., USD)
+        to_currency: Target currency code (e.g., EUR)
+        
+    Returns:
+        TextContent with conversion result
+    """
+    try:
+        logging.info(f"💱 Converting {amount} {from_currency} to {to_currency}")
+        
+        # Using free exchange rate API
+        url = f"https://api.exchangerate-api.com/v4/latest/{from_currency}"
+        
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()
+        
+        data = response.json()
+        
+        if to_currency not in data["rates"]:
+            raise ValueError(f"Currency not found: {to_currency}")
+        
+        rate = data["rates"][to_currency]
+        converted_amount = amount * rate
+        
+        result = {
+            "amount": amount,
+            "from_currency": from_currency,
+            "to_currency": to_currency,
+            "exchange_rate": rate,
+            "converted_amount": converted_amount,
+            "timestamp": data.get("date", datetime.now().isoformat())
+        }
+        
+        logging.info(f"✅ {amount} {from_currency} = {converted_amount:.2f} {to_currency}")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"rate": rate}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Currency conversion failed: {str(e)}"
+        logging.error(f"Currency error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "currency_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def search_wikipedia(
+    query: str,
+    language: str = "en",
+    sentences: int = 5
+) -> Union[str, TextContent]:
+    """
+    Search Wikipedia and get article summary.
+    
+    Args:
+        query: Search query
+        language: Wikipedia language (en, zh, etc.)
+        sentences: Number of sentences in summary
+        
+    Returns:
+        TextContent with Wikipedia article
+    """
+    try:
+        wikipedia.set_lang(language)
+        
+        logging.info(f"📚 Searching Wikipedia for: {query}")
+        
+        # Search for pages
+        search_results = wikipedia.search(query, results=3)
+        
+        if not search_results:
+            raise ValueError(f"No Wikipedia articles found for: {query}")
+        
+        # Get the first result's page
+        page = wikipedia.page(search_results[0], auto_suggest=False)
+        
+        summary = wikipedia.summary(search_results[0], sentences=sentences, auto_suggest=False)
+        
+        result = {
+            "title": page.title,
+            "url": page.url,
+            "summary": summary,
+            "language": language,
+            "search_results": search_results
+        }
+        
+        logging.info(f"✅ Found Wikipedia article: {page.title}")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"query": query, "language": language}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Wikipedia search failed: {str(e)}"
+        logging.error(f"Wikipedia error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "wikipedia_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def search_arxiv(
+    query: str,
+    max_results: int = 5,
+    sort_by: str = "relevance"
+) -> Union[str, TextContent]:
+    """
+    Search ArXiv for academic papers.
+    
+    Args:
+        query: Search query
+        max_results: Maximum number of results
+        sort_by: Sort method (relevance, lastUpdatedDate, submittedDate)
+        
+    Returns:
+        TextContent with ArXiv papers
+    """
+    try:
+        import arxiv
+        
+        logging.info(f"🔬 Searching ArXiv for: {query}")
+        
+        # Map sort_by to arxiv.SortCriterion
+        sort_map = {
+            "relevance": arxiv.SortCriterion.Relevance,
+            "lastUpdatedDate": arxiv.SortCriterion.LastUpdatedDate,
+            "submittedDate": arxiv.SortCriterion.SubmittedDate
+        }
+        
+        sort_criterion = sort_map.get(sort_by, arxiv.SortCriterion.Relevance)
+        
+        search = arxiv.Search(
+            query=query,
+            max_results=max_results,
+            sort_by=sort_criterion
+        )
+
+        # arxiv.Client defaults to page_size=100 even when the caller asks for
+        # only a handful of papers.  That needlessly expands the official API
+        # request and made paired experiment arms much more susceptible to
+        # export.arxiv.org throttling.  Keep the request page bounded by the
+        # public MCP argument while retaining the library's documented delay
+        # and retry behavior.
+        client = arxiv.Client(
+            page_size=max(1, min(max_results, 100)),
+            delay_seconds=3.0,
+            num_retries=3,
+        )
+        
+        papers = []
+        for result in client.results(search):
+            papers.append({
+                "title": result.title,
+                "authors": [author.name for author in result.authors],
+                "summary": result.summary[:500] + "...",
+                "published": result.published.isoformat(),
+                "url": result.entry_id,
+                "pdf_url": result.pdf_url,
+                "categories": result.categories
+            })
+        
+        logging.info(f"✅ Found {len(papers)} papers")
+        
+        action_response = ActionResponse(
+            success=True,
+            message={
+                "query": query,
+                "papers": papers,
+                "count": len(papers)
+            },
+            metadata={"query": query, "max_results": max_results}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"ArXiv search failed: {str(e)}"
+        logging.error(f"ArXiv error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "arxiv_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def search_wayback(
+    url: str,
+    year: int | None = None,
+    limit: int = 10
+) -> Union[str, TextContent]:
+    """
+    Search Wayback Machine for archived versions of a URL.
+    
+    Args:
+        url: URL to search for
+        year: Optional year to filter results
+        limit: Maximum number of snapshots to return
+        
+    Returns:
+        TextContent with archived snapshots
+    """
+    try:
+        logging.info(f"🕰️ Searching Wayback Machine for: {url}")
+        
+        # CDX API endpoint
+        cdx_url = "http://web.archive.org/cdx/search/cdx"
+        params = {
+            "url": url,
+            "output": "json",
+            "limit": limit,
+            "fl": "timestamp,original,statuscode,mimetype"
+        }
+        
+        if year:
+            params["from"] = f"{year}0101"
+            params["to"] = f"{year}1231"
+        
+        response = requests.get(cdx_url, params=params, timeout=30)
+        response.raise_for_status()
+        
+        data = response.json()
+        
+        # First row is headers
+        if len(data) <= 1:
+            raise ValueError(f"No archived snapshots found for: {url}")
+        
+        headers = data[0]
+        snapshots = []
+        
+        for row in data[1:]:
+            snapshot = dict(zip(headers, row))
+            # Convert timestamp to readable format
+            ts = snapshot["timestamp"]
+            dt = datetime.strptime(ts, "%Y%m%d%H%M%S")
+            
+            snapshots.append({
+                "timestamp": dt.isoformat(),
+                "url": f"https://web.archive.org/web/{ts}/{snapshot['original']}",
+                "status_code": snapshot.get("statuscode"),
+                "mime_type": snapshot.get("mimetype")
+            })
+        
+        logging.info(f"✅ Found {len(snapshots)} archived snapshots")
+        
+        action_response = ActionResponse(
+            success=True,
+            message={
+                "url": url,
+                "snapshots": snapshots,
+                "count": len(snapshots)
+            },
+            metadata={"url": url, "year": year}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Wayback Machine search failed: {str(e)}"
+        logging.error(f"Wayback error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "wayback_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_crypto_price(
+    symbol: str,
+    vs_currency: str = "usd"
+) -> Union[str, TextContent]:
+    """
+    Get cryptocurrency price information using CoinGecko API (free, no API key required).
+    
+    Args:
+        symbol: Cryptocurrency symbol or ID (e.g., bitcoin, ethereum, btc, eth)
+        vs_currency: Target currency (usd, eur, gbp, etc.)
+        
+    Returns:
+        TextContent with cryptocurrency data
+    """
+    try:
+        logging.info(f"💰 Getting crypto price for: {symbol}")
+        
+        # CoinGecko free API
+        # First, try to get the coin ID from symbol
+        symbol_lower = symbol.lower()
+        
+        # Map common symbols to CoinGecko IDs
+        symbol_map = {
+            "btc": "bitcoin",
+            "eth": "ethereum",
+            "usdt": "tether",
+            "bnb": "binancecoin",
+            "sol": "solana",
+            "xrp": "ripple",
+            "usdc": "usd-coin",
+            "ada": "cardano",
+            "doge": "dogecoin",
+            "trx": "tron",
+            "dot": "polkadot",
+            "matic": "matic-network",
+            "dai": "dai",
+            "shib": "shiba-inu",
+            "avax": "avalanche-2"
+        }
+        
+        # Use mapped ID or try the symbol directly
+        coin_id = symbol_map.get(symbol_lower, symbol_lower)
+        
+        # Get price data from CoinGecko
+        url = "https://api.coingecko.com/api/v3/simple/price"
+        params = {
+            "ids": coin_id,
+            "vs_currencies": vs_currency,
+            "include_market_cap": "true",
+            "include_24hr_vol": "true",
+            "include_24hr_change": "true",
+            "include_last_updated_at": "true"
+        }
+        
+        headers = {
+            "User-Agent": "Mozilla/5.0"
+        }
+        
+        response = requests.get(url, params=params, headers=headers, timeout=10)
+        response.raise_for_status()
+        
+        data = response.json()
+        
+        if not data or coin_id not in data:
+            raise ValueError(f"Cryptocurrency not found: {symbol}")
+        
+        coin_data = data[coin_id]
+        
+        result = {
+            "symbol": symbol.upper(),
+            "coin_id": coin_id,
+            "currency": vs_currency.upper(),
+            "current_price": coin_data.get(vs_currency),
+            "market_cap": coin_data.get(f"{vs_currency}_market_cap"),
+            "volume_24h": coin_data.get(f"{vs_currency}_24h_vol"),
+            "price_change_24h_percent": coin_data.get(f"{vs_currency}_24h_change"),
+            "last_updated": datetime.fromtimestamp(coin_data.get("last_updated_at", 0)).isoformat() if coin_data.get("last_updated_at") else None,
+            "provider": "CoinGecko"
+        }
+        
+        logging.info(f"✅ Crypto price: {result['current_price']} {vs_currency.upper()}")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"symbol": symbol, "provider": "CoinGecko", "api_key_required": False}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Crypto price query failed: {str(e)}"
+        logging.error(f"Crypto error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "crypto_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def search_location(
+    query: str,
+    limit: int = 5,
+    country_code: str | None = None
+) -> Union[str, TextContent]:
+    """
+    Search for locations using Nominatim (OpenStreetMap) API (free, no API key required).
+    
+    Args:
+        query: Location query (e.g., "Eiffel Tower", "New York", "coffee shop near me")
+        limit: Maximum number of results (1-50)
+        country_code: Optional country code filter (e.g., "us", "gb", "fr")
+        
+    Returns:
+        TextContent with location search results
+    """
+    try:
+        logging.info(f"📍 Searching location: {query}")
+        
+        # Nominatim API (OpenStreetMap)
+        url = "https://nominatim.openstreetmap.org/search"
+        
+        params = {
+            "q": query,
+            "format": "json",
+            "limit": min(limit, 50),
+            "addressdetails": 1,
+            "extratags": 1
+        }
+        
+        if country_code:
+            params["countrycodes"] = country_code.lower()
+        
+        headers = {
+            "User-Agent": "PerceptionToolsMCP/1.0"
+        }
+        
+        response = requests.get(url, params=params, headers=headers, timeout=10)
+        response.raise_for_status()
+        
+        data = response.json()
+        
+        if not data:
+            raise ValueError(f"No locations found for: {query}")
+        
+        locations = []
+        for item in data:
+            address = item.get("address", {})
+            
+            locations.append({
+                "display_name": item.get("display_name"),
+                "latitude": float(item.get("lat")),
+                "longitude": float(item.get("lon")),
+                "type": item.get("type"),
+                "category": item.get("class"),
+                "address": {
+                    "country": address.get("country"),
+                    "country_code": address.get("country_code"),
+                    "state": address.get("state"),
+                    "city": address.get("city") or address.get("town") or address.get("village"),
+                    "postcode": address.get("postcode"),
+                    "road": address.get("road")
+                },
+                "importance": item.get("importance"),
+                "osm_id": item.get("osm_id"),
+                "osm_type": item.get("osm_type")
+            })
+        
+        logging.info(f"✅ Found {len(locations)} locations")
+        
+        action_response = ActionResponse(
+            success=True,
+            message={
+                "query": query,
+                "locations": locations,
+                "count": len(locations)
+            },
+            metadata={"provider": "Nominatim (OpenStreetMap)", "api_key_required": False}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Location search failed: {str(e)}"
+        logging.error(f"Location search error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "location_search_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def search_poi(
+    query: str,
+    latitude: float,
+    longitude: float,
+    radius: int = 1000,
+    limit: int = 10
+) -> Union[str, TextContent]:
+    """
+    Search for Points of Interest (POI) near a location using Overpass API (OpenStreetMap).
+    Free, no API key required.
+    
+    Args:
+        query: Type of POI (e.g., "restaurant", "cafe", "hospital", "atm", "hotel")
+        latitude: Center latitude
+        longitude: Center longitude
+        radius: Search radius in meters (default: 1000)
+        limit: Maximum number of results (default: 10)
+        
+    Returns:
+        TextContent with POI search results
+    """
+    try:
+        logging.info(f"🔍 Searching POIs: {query} near ({latitude}, {longitude})")
+        
+        # Overpass API query
+        # Search for amenities, shops, tourism, etc.
+        overpass_query = f"""
+        [out:json][timeout:10];
+        (
+          node["amenity"~"{query}",i](around:{radius},{latitude},{longitude});
+          node["shop"~"{query}",i](around:{radius},{latitude},{longitude});
+          node["tourism"~"{query}",i](around:{radius},{latitude},{longitude});
+          node["name"~"{query}",i](around:{radius},{latitude},{longitude});
+        );
+        out body {limit};
+        """
+        
+        url = "https://overpass-api.de/api/interpreter"
+        
+        response = requests.post(url, data={"data": overpass_query}, timeout=30)
+        response.raise_for_status()
+        
+        data = response.json()
+        
+        elements = data.get("elements", [])
+        
+        if not elements:
+            raise ValueError(f"No POIs found for '{query}' near the specified location")
+        
+        pois = []
+        for element in elements[:limit]:
+            tags = element.get("tags", {})
+            
+            pois.append({
+                "name": tags.get("name", "Unnamed"),
+                "type": tags.get("amenity") or tags.get("shop") or tags.get("tourism") or "unknown",
+                "latitude": element.get("lat"),
+                "longitude": element.get("lon"),
+                "address": tags.get("addr:street"),
+                "city": tags.get("addr:city"),
+                "postcode": tags.get("addr:postcode"),
+                "phone": tags.get("phone"),
+                "website": tags.get("website"),
+                "opening_hours": tags.get("opening_hours"),
+                "cuisine": tags.get("cuisine"),
+                "osm_id": element.get("id"),
+                "osm_type": element.get("type")
+            })
+        
+        logging.info(f"✅ Found {len(pois)} POIs")
+        
+        action_response = ActionResponse(
+            success=True,
+            message={
+                "query": query,
+                "center": {"latitude": latitude, "longitude": longitude},
+                "radius_meters": radius,
+                "pois": pois,
+                "count": len(pois)
+            },
+            metadata={"provider": "Overpass API (OpenStreetMap)", "api_key_required": False}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"POI search failed: {str(e)}"
+        logging.error(f"POI search error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "poi_search_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )

chapter4/perception-tools/src/search_tools.py

@@ -0,0 +1,431 @@
+"""
+Search tools: knowledge base, web search, and file download.
+"""
+import json
+import logging
+import os
+import time
+import traceback
+from pathlib import Path
+from typing import Union
+
+import requests
+from bs4 import BeautifulSoup
+from dotenv import load_dotenv
+from mcp.types import TextContent
+from pydantic import BaseModel, Field
+
+from base import ActionResponse, is_url, download_file_from_url
+
+
+load_dotenv()
+
+
+class SearchResult(BaseModel):
+    """Individual search result with structured data."""
+    
+    id: str
+    title: str
+    url: str
+    snippet: str
+    source: str
+
+
+class SearchMetadata(BaseModel):
+    """Metadata for search operations."""
+    
+    query: str
+    search_engine: str
+    total_results: int
+    search_time: float | None = None
+    language: str = "en"
+    country: str = "us"
+
+
+async def search_web(
+    query: str,
+    num_results: int = 5,
+    region: str = "wt-wt"
+) -> Union[str, TextContent]:
+    """
+    Search the web using DuckDuckGo (free, no API key required).
+    
+    Args:
+        query: The search query string
+        num_results: Number of results to return (1-10)
+        region: Region code (e.g., 'us-en', 'uk-en', 'wt-wt' for worldwide)
+        
+    Returns:
+        TextContent with search results
+    """
+    try:
+        if not query or not query.strip():
+            raise ValueError("Search query cannot be empty")
+
+        if num_results <= 0:
+            metadata = SearchMetadata(
+                query=query,
+                search_engine="none",
+                total_results=0,
+                search_time=0.0,
+                language="en",
+                country=region,
+            )
+            action_response = ActionResponse(
+                success=True,
+                message={
+                    "query": query,
+                    "results": [],
+                    "count": 0,
+                },
+                metadata=metadata.model_dump(),
+            )
+            return TextContent(
+                type="text",
+                text=json.dumps(action_response.model_dump()),
+            )
+        
+        validated_num_results = max(1, min(num_results, 10))
+        
+        logging.info(f"🔍 Searching for: '{query}'")
+        start_time = time.time()
+        
+        # Use DuckDuckGo HTML version for scraping
+        url = "https://html.duckduckgo.com/html/"
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+        }
+        
+        data = {
+            "q": query.strip(),
+            "kl": region
+        }
+        
+        search_results = []
+        search_engine = "duckduckgo"
+        provider_errors = []
+        try:
+            response = requests.post(url, data=data, headers=headers, timeout=15)
+            response.raise_for_status()
+            soup = BeautifulSoup(response.text, 'html.parser')
+            result_divs = soup.find_all('div', class_='result')
+            for i, result_div in enumerate(result_divs[:validated_num_results]):
+                try:
+                    title_tag = result_div.find('a', class_='result__a')
+                    if not title_tag:
+                        continue
+                    snippet_tag = result_div.find('a', class_='result__snippet')
+                    search_results.append(SearchResult(
+                        id=f"ddg-{i}",
+                        title=title_tag.get_text(strip=True),
+                        url=title_tag.get('href', ''),
+                        snippet=(snippet_tag.get_text(strip=True)
+                                 if snippet_tag else ""),
+                        source="duckduckgo",
+                    ))
+                except Exception as exc:
+                    logging.warning("Error parsing DuckDuckGo result %s: %s", i, exc)
+        except Exception as exc:
+            provider_errors.append(f"duckduckgo-html:{type(exc).__name__}")
+            logging.warning("DuckDuckGo HTML search failed: %s", exc)
+
+        # DuckDuckGo occasionally serves an HTML variant without ``div.result``
+        # while still returning HTTP 200. Fall back to its public Lite result
+        # page so an empty parser match cannot masquerade as a successful
+        # current-web search.
+        if not search_results:
+            try:
+                lite_response = requests.post(
+                    "https://lite.duckduckgo.com/lite/",
+                    data=data,
+                    headers=headers,
+                    timeout=15,
+                )
+                lite_response.raise_for_status()
+                lite_soup = BeautifulSoup(lite_response.text, "html.parser")
+                for i, link in enumerate(
+                    lite_soup.select("a.result-link")[:validated_num_results]
+                ):
+                    snippet_tag = link.find_next(class_="result-snippet")
+                    search_results.append(SearchResult(
+                        id=f"ddg-lite-{i}",
+                        title=link.get_text(" ", strip=True),
+                        url=link.get("href", ""),
+                        snippet=(snippet_tag.get_text(" ", strip=True)
+                                 if snippet_tag else ""),
+                        source="duckduckgo-lite",
+                    ))
+                if search_results:
+                    search_engine = "duckduckgo-lite"
+            except Exception as exc:
+                provider_errors.append(f"duckduckgo-lite:{type(exc).__name__}")
+                logging.warning("DuckDuckGo Lite search failed: %s", exc)
+
+        if not search_results:
+            serper_key = os.getenv("SERPER_API_KEY")
+            tavily_key = os.getenv("TAVILY_API_KEY")
+            if serper_key:
+                try:
+                    serper = requests.post(
+                        "https://google.serper.dev/search",
+                        headers={"X-API-KEY": serper_key,
+                                 "Content-Type": "application/json"},
+                        json={"q": query.strip(), "num": validated_num_results},
+                        timeout=20,
+                    )
+                    serper.raise_for_status()
+                    for i, row in enumerate(
+                        serper.json().get("organic", [])[:validated_num_results]
+                    ):
+                        search_results.append(SearchResult(
+                            id=f"serper-{i}", title=row.get("title", ""),
+                            url=row.get("link", ""), snippet=row.get("snippet", ""),
+                            source="serper-google",
+                        ))
+                    if search_results:
+                        search_engine = "serper-google"
+                except Exception as exc:
+                    provider_errors.append(f"serper:{type(exc).__name__}")
+                    logging.warning("Serper search failed; trying next provider: %s", exc)
+            if not search_results and tavily_key:
+                try:
+                    tavily = requests.post(
+                        "https://api.tavily.com/search",
+                        json={"api_key": tavily_key, "query": query.strip(),
+                              "max_results": validated_num_results,
+                              "search_depth": "basic", "include_answer": False},
+                        timeout=20,
+                    )
+                    tavily.raise_for_status()
+                    for i, row in enumerate(
+                        tavily.json().get("results", [])[:validated_num_results]
+                    ):
+                        search_results.append(SearchResult(
+                            id=f"tavily-{i}", title=row.get("title", ""),
+                            url=row.get("url", ""), snippet=row.get("content", ""),
+                            source="tavily",
+                        ))
+                    if search_results:
+                        search_engine = "tavily"
+                except Exception as exc:
+                    provider_errors.append(f"tavily:{type(exc).__name__}")
+                    logging.warning("Tavily search failed: %s", exc)
+
+        if not search_results:
+            raise LookupError(
+                "No configured live search provider returned results; attempts="
+                + ",".join(provider_errors)
+            )
+        search_time = time.time() - start_time
+        
+        metadata = SearchMetadata(
+            query=query,
+            search_engine=search_engine,
+            total_results=len(search_results),
+            search_time=search_time,
+            language="en",
+            country=region
+        )
+        
+        formatted_content = {
+            "query": query,
+            "results": [result.model_dump() for result in search_results],
+            "count": len(search_results)
+        }
+        
+        logging.info(f"✅ Found {len(search_results)} results in {search_time:.2f}s")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=formatted_content,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Search operation failed: {str(e)}"
+        logging.error(f"Search error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "search_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def download_file(
+    url: str,
+    output_path: str,
+    overwrite: bool = False,
+    timeout: int = 180
+) -> Union[str, TextContent]:
+    """
+    Download a file from a URL.
+    
+    Args:
+        url: URL to download from
+        output_path: Local path to save the file
+        overwrite: Whether to overwrite existing files
+        timeout: Download timeout in seconds
+        
+    Returns:
+        TextContent with download result
+    """
+    try:
+        if not url.startswith(("http://", "https://")):
+            raise ValueError("Only HTTP/HTTPS URLs are supported")
+        
+        output_file = Path(output_path).expanduser().resolve()
+        
+        if output_file.exists() and not overwrite:
+            raise ValueError(f"File already exists: {output_file}. Use overwrite=True to replace.")
+        
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+        
+        logging.info(f"📥 Downloading from: {url}")
+        start_time = time.time()
+        
+        temp_path, content = download_file_from_url(url, timeout=timeout)
+        
+        # Move to final destination
+        with open(output_file, 'wb') as f:
+            f.write(content)
+        
+        # Clean up temp file
+        Path(temp_path).unlink(missing_ok=True)
+        
+        duration = time.time() - start_time
+        file_size = len(content)
+        
+        logging.info(f"✅ Downloaded {file_size / 1024:.2f} KB in {duration:.2f}s")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=f"Successfully downloaded file to {output_file}",
+            metadata={
+                "url": url,
+                "output_path": str(output_file),
+                "file_size_bytes": file_size,
+                "duration_seconds": duration
+            }
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Download failed: {str(e)}"
+        logging.error(f"Download error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "download_error", "url": url}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def search_knowledge_base(
+    query: str,
+    knowledge_base_path: str,
+    top_k: int = 5
+) -> Union[str, TextContent]:
+    """
+    Search a local knowledge base using simple text matching.
+    
+    Args:
+        query: Search query
+        knowledge_base_path: Path to knowledge base directory
+        top_k: Number of top results to return
+        
+    Returns:
+        TextContent with search results
+    """
+    try:
+        kb_path = Path(knowledge_base_path).expanduser().resolve()
+        
+        if not kb_path.exists():
+            raise FileNotFoundError(f"Knowledge base not found: {kb_path}")
+        
+        if not kb_path.is_dir():
+            raise ValueError(f"Knowledge base path must be a directory: {kb_path}")
+        
+        logging.info(f"🔍 Searching knowledge base: {kb_path}")
+        
+        # Simple file search - find files containing the query
+        results = []
+        query_lower = query.lower()
+        
+        for file_path in kb_path.rglob("*"):
+            if file_path.is_file() and file_path.suffix in [".txt", ".md", ".json"]:
+                try:
+                    content = file_path.read_text(encoding="utf-8", errors="ignore")
+                    if query_lower in content.lower():
+                        # Get snippet around first occurrence
+                        idx = content.lower().index(query_lower)
+                        start = max(0, idx - 100)
+                        end = min(len(content), idx + 200)
+                        snippet = content[start:end].strip()
+                        
+                        results.append({
+                            "file": str(file_path.relative_to(kb_path)),
+                            "snippet": snippet,
+                            "relevance": content.lower().count(query_lower)
+                        })
+                except Exception as e:
+                    logging.warning(f"Error reading {file_path}: {e}")
+                    continue
+        
+        # Sort by relevance and limit
+        results.sort(key=lambda x: x["relevance"], reverse=True)
+        results = results[:max(0, top_k)]
+        
+        logging.info(f"✅ Found {len(results)} results")
+        
+        action_response = ActionResponse(
+            success=True,
+            message={
+                "query": query,
+                "results": results,
+                "total_found": len(results)
+            },
+            metadata={
+                "knowledge_base": str(kb_path),
+                "top_k": top_k
+            }
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Knowledge base search failed: {str(e)}"
+        logging.error(f"KB search error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "kb_search_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )

chapter4/perception-tools/src/test_csv_max_rows.py

@@ -0,0 +1,20 @@
+"""Regression: extract_csv_content must honor max_rows in data, not hard-cap at 100."""
+import json
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+from document_processing_tools import extract_csv_content
+
+
+@pytest.mark.asyncio
+async def test_csv_data_honors_max_rows(tmp_path: Path):
+    path = tmp_path / "t.csv"
+    pd.DataFrame({"id": range(250)}).to_csv(path, index=False)
+    r = await extract_csv_content(str(path), max_rows=1000)
+    payload = json.loads(r.text)
+    msg = payload["message"]
+    assert msg["rows"] == 250
+    assert len(msg["data"]) == 250
+    assert msg["truncated"] is False

chapter4/perception-tools/src/test_grep_max_results_zero.py

@@ -0,0 +1,59 @@
+"""Regression: grep_search(max_results=0) must return zero hits, not one."""
+import json
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+
+def _stub():
+    for name in ["dotenv", "requests", "mcp", "mcp.types", "mcp.server"]:
+        sys.modules.setdefault(name, types.ModuleType(name))
+    sys.modules["dotenv"].load_dotenv = lambda *a, **k: None
+
+    class TextContent:
+        def __init__(self, type=None, text=None):
+            self.type = type
+            self.text = text
+
+    sys.modules["mcp.types"].TextContent = TextContent
+
+    class MCPServer:
+        def __init__(self, *a, **k):
+            pass
+
+        def tool(self, *a, **k):
+            def deco(fn):
+                return fn
+
+            return deco
+
+    sys.modules["mcp.server"].MCPServer = MCPServer
+
+
+_stub()
+from filesystem_tools import grep_search  # noqa: E402
+
+
+@pytest.mark.asyncio
+async def test_max_results_zero_returns_no_matches(tmp_path: Path):
+    (tmp_path / "a.py").write_text("hello world\nhello again\n", encoding="utf-8")
+    r = await grep_search("hello", str(tmp_path), max_results=0)
+    payload = json.loads(r.text if hasattr(r, "text") else r)
+    assert payload["success"] is True
+    msg = payload["message"]
+    assert msg["results"] == []
+    assert msg["total_found"] == 0
+    assert msg["truncated"] is False
+
+
+@pytest.mark.asyncio
+async def test_max_results_one_still_caps(tmp_path: Path):
+    (tmp_path / "a.py").write_text("hello world\nhello again\n", encoding="utf-8")
+    r = await grep_search("hello", str(tmp_path), max_results=1)
+    payload = json.loads(r.text if hasattr(r, "text") else r)
+    msg = payload["message"]
+    assert msg["total_found"] == 1
+    assert len(msg["results"]) == 1
+    assert msg["truncated"] is True

chapter4/perception-tools/src/test_negative_max_length.py

@@ -0,0 +1,45 @@
+"""Regression: negative max_length must not drop the last character."""
+import asyncio
+import json
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+
+def _stub():
+    for name in ["dotenv", "requests", "mcp", "mcp.types", "mcp.server"]:
+        sys.modules.setdefault(name, types.ModuleType(name))
+    sys.modules["dotenv"].load_dotenv = lambda *a, **k: None
+    class TextContent:
+        def __init__(self, type=None, text=None):
+            self.type = type
+            self.text = text
+    sys.modules["mcp.types"].TextContent = TextContent
+    class MCPServer:
+        def __init__(self, *a, **k): pass
+        def tool(self, *a, **k):
+            def deco(fn): return fn
+            return deco
+    sys.modules["mcp.server"].MCPServer = MCPServer
+
+
+_stub()
+from filesystem_tools import read_file  # noqa: E402
+
+
+@pytest.mark.asyncio
+async def test_negative_max_length_keeps_full_content(tmp_path: Path):
+    path = tmp_path / "a.txt"
+    path.write_text("hello world", encoding="utf-8")
+    r = await read_file(str(path), max_length=-1)
+    payload = json.loads(r.text if hasattr(r, "text") else r)
+    msg = payload.get("message", payload)
+    if isinstance(msg, dict) and "content" in msg:
+        content = msg["content"]
+    elif isinstance(msg, str):
+        content = msg
+    else:
+        content = str(msg)
+    assert "hello world" in content

chapter4/perception-tools/src/test_search_web_num_results_zero.py

@@ -0,0 +1,66 @@
+"""Regression test for search_web num_results=0 handling.
+
+Proves contract: Requesting zero search results short-circuits external search
+providers and returns success with empty result list and count 0.
+Locks out bug where max(1, min(num_results, 10)) clamped num_results=0 to 1.
+"""
+
+import json
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent))
+
+def _stub():
+    for name in ["dotenv", "mcp", "mcp.types", "mcp.server", "mcp.server.fastmcp"]:
+        sys.modules.setdefault(name, types.ModuleType(name))
+    sys.modules["dotenv"].load_dotenv = lambda *a, **k: None
+
+    class TextContent:
+        def __init__(self, type=None, text=None):
+            self.type = type
+            self.text = text
+
+    sys.modules["mcp.types"].TextContent = TextContent
+
+    class FastMCP:
+        def __init__(self, *a, **k):
+            pass
+
+        def tool(self, *a, **k):
+            def deco(fn):
+                return fn
+
+            return deco
+
+    sys.modules["mcp.server.fastmcp"].FastMCP = FastMCP
+
+
+_stub()
+
+from search_tools import search_web  # noqa: E402
+
+
+@pytest.mark.asyncio
+async def test_search_web_num_results_zero_returns_no_results():
+    result = await search_web("Python", num_results=0)
+    payload = json.loads(result.text if hasattr(result, "text") else result)
+    assert payload["success"] is True
+    message = payload["message"]
+    assert message["results"] == []
+    assert message["count"] == 0
+    assert payload["metadata"]["total_results"] == 0
+
+
+@pytest.mark.asyncio
+async def test_search_web_num_results_negative_returns_no_results():
+    result = await search_web("Python", num_results=-5)
+    payload = json.loads(result.text if hasattr(result, "text") else result)
+    assert payload["success"] is True
+    message = payload["message"]
+    assert message["results"] == []
+    assert message["count"] == 0
+    assert payload["metadata"]["total_results"] == 0

chapter4/perception-tools/src/wayback_enhanced.py

@@ -0,0 +1,83 @@
+"""
+Enhanced Wayback Machine tools.
+"""
+import json
+import logging
+import traceback
+from typing import Union
+
+import requests
+from bs4 import BeautifulSoup
+from dotenv import load_dotenv
+from mcp.types import TextContent
+from waybackpy import WaybackMachineCDXServerAPI
+
+from base import ActionResponse
+
+
+load_dotenv()
+
+
+async def get_archived_content(
+    url: str,
+    timestamp: str
+) -> Union[str, TextContent]:
+    """
+    Get content from archived webpage.
+    
+    Args:
+        url: URL to retrieve
+        timestamp: Wayback timestamp (YYYYMMDDhhmmss)
+        
+    Returns:
+        TextContent with archived content
+    """
+    try:
+        logging.info(f"🕰️ Getting archived content: {url} at {timestamp}")
+        
+        # Query for closest snapshot
+        cdx_api = WaybackMachineCDXServerAPI(url)
+        snapshot = cdx_api.near(wayback_machine_timestamp=timestamp)
+        
+        if not snapshot:
+            raise ValueError("No archived version found")
+        
+        # Fetch content
+        response = requests.get(snapshot.archive_url, timeout=30)
+        response.raise_for_status()
+        
+        # Extract text
+        soup = BeautifulSoup(response.content, 'html.parser')
+        text = soup.get_text(separator=" ", strip=True)
+        
+        result = {
+            "url": url,
+            "timestamp": timestamp,
+            "actual_timestamp": snapshot.timestamp,
+            "archive_url": snapshot.archive_url,
+            "content": text[:10000],  # Limit to 10k chars
+            "content_length": len(text)
+        }
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"url": url}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        action_response = ActionResponse(
+            success=False,
+            message=f"Failed: {str(e)}",
+            metadata={"error_type": "wayback_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )

chapter4/perception-tools/src/wiki_enhanced.py

@@ -0,0 +1,269 @@
+"""
+Enhanced Wikipedia tools with full article access.
+Based on AWorld wiki-server complete implementation.
+"""
+import json
+import logging
+import traceback
+import calendar
+from datetime import datetime
+from typing import Union
+
+import requests
+import wikipedia
+from dotenv import load_dotenv
+from mcp.types import TextContent
+
+from base import ActionResponse
+
+
+load_dotenv()
+
+
+async def get_article_content(
+    title: str,
+    language: str = "en"
+) -> Union[str, TextContent]:
+    """
+    Get full Wikipedia article content.
+    
+    Args:
+        title: Article title
+        language: Language code
+        
+    Returns:
+        TextContent with full article
+    """
+    try:
+        wikipedia.set_lang(language)
+        
+        logging.info(f"📚 Getting full article: {title}")
+        
+        page = wikipedia.page(title, auto_suggest=True)
+        
+        result = {
+            "title": page.title,
+            "url": page.url,
+            "content": page.content,
+            "summary": page.summary,
+            "categories": page.categories[:20] if page.categories else [],
+            "links": page.links[:50] if page.links else [],
+            "images": page.images[:10] if page.images else []
+        }
+        
+        logging.info(f"✅ Retrieved article: {len(page.content)} chars")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"language": language, "title": page.title}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Failed to get article: {str(e)}"
+        logging.error(f"Wiki error: {traceback.format_exc()}")
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata={"error_type": "wiki_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_article_categories(
+    title: str,
+    language: str = "en"
+) -> Union[str, TextContent]:
+    """
+    Get categories for Wikipedia article.
+    
+    Args:
+        title: Article title
+        language: Language code
+        
+    Returns:
+        TextContent with categories
+    """
+    try:
+        wikipedia.set_lang(language)
+        page = wikipedia.page(title, auto_suggest=True)
+        
+        result = {
+            "title": page.title,
+            "categories": page.categories,
+            "count": len(page.categories)
+        }
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"language": language}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        action_response = ActionResponse(
+            success=False,
+            message=f"Failed: {str(e)}",
+            metadata={"error_type": "wiki_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_article_links(
+    title: str,
+    language: str = "en"
+) -> Union[str, TextContent]:
+    """
+    Get links from Wikipedia article.
+    
+    Args:
+        title: Article title
+        language: Language code
+        
+    Returns:
+        TextContent with links
+    """
+    try:
+        wikipedia.set_lang(language)
+        page = wikipedia.page(title, auto_suggest=True)
+        
+        result = {
+            "title": page.title,
+            "links": page.links[:100],  # Limit to 100
+            "total_links": len(page.links),
+            "count": min(100, len(page.links))
+        }
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata={"language": language}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        action_response = ActionResponse(
+            success=False,
+            message=f"Failed: {str(e)}",
+            metadata={"error_type": "wiki_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_article_history(
+    title: str,
+    date: str,
+    language: str = "en"
+) -> Union[str, TextContent]:
+    """
+    Get historical version of Wikipedia article.
+    
+    Args:
+        title: Article title
+        date: Target date (YYYY/MM/DD)
+        language: Language code
+        
+    Returns:
+        TextContent with historical content
+    """
+    try:
+        logging.info(f"📚 Getting historical version: {title} at {date}")
+        
+        # Parse date (YYYY/MM or YYYY/MM/DD)
+        if not isinstance(date, str) or "/" not in date:
+            raise ValueError("date must be YYYY/MM/DD or YYYY/MM")
+        date_parts = date.split("/")
+        if len(date_parts) < 2:
+            raise ValueError("date must be YYYY/MM/DD or YYYY/MM")
+        year = int(date_parts[0])
+        month = int(date_parts[1])
+        day = int(date_parts[2]) if len(date_parts) > 2 else calendar.monthrange(year, month)[1]
+        
+        target_date = datetime(year, month, day)
+        
+        # Get page revisions via Wikipedia API
+        params = {
+            "action": "query",
+            "prop": "revisions",
+            "titles": title,
+            "rvprop": "ids|timestamp|user|comment|content",
+            "rvlimit": 1,
+            "rvdir": "older",
+            "rvstart": target_date.isoformat(),
+            "format": "json"
+        }
+        
+        api_url = f"https://{language}.wikipedia.org/w/api.php"
+        response = requests.get(api_url, params=params, timeout=10)
+        data = response.json()
+        
+        page = next(iter(data["query"]["pages"].values()))
+        
+        if "revisions" in page:
+            revision = page["revisions"][0]
+            actual_date = datetime.fromisoformat(revision["timestamp"].replace("Z", "+00:00"))
+            
+            result = {
+                "title": title,
+                "requested_date": date,
+                "actual_date": actual_date.strftime("%Y/%m/%d"),
+                "content": revision["*"],
+                "editor": revision["user"],
+                "comment": revision.get("comment", "")
+            }
+            
+            action_response = ActionResponse(
+                success=True,
+                message=result,
+                metadata={"language": language}
+            )
+        else:
+            action_response = ActionResponse(
+                success=False,
+                message="No historical version found",
+                metadata={"error_type": "not_found"}
+            )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        action_response = ActionResponse(
+            success=False,
+            message=f"Failed: {str(e)}",
+            metadata={"error_type": "wiki_error"}
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )

chapter4/perception-tools/src/yahoo_finance_tools.py

@@ -0,0 +1,437 @@
+"""
+Yahoo Finance comprehensive tools.
+Based on AWorld MCP server implementation.
+Provides stock quotes, historical data, company info, and financial statements.
+"""
+import json
+import logging
+import time
+import traceback
+from datetime import datetime
+from typing import Union, Literal
+
+import yfinance as yf
+from dotenv import load_dotenv
+from mcp.types import TextContent
+from pydantic import BaseModel, Field
+
+from base import ActionResponse
+
+
+load_dotenv()
+
+
+class YFinanceMetadata(BaseModel):
+    """Metadata for Yahoo Finance operation results."""
+    
+    symbol: str
+    operation: str
+    execution_time: float | None = None
+    data_points: int | None = None
+    error_type: str | None = None
+    timestamp: str | None = None
+
+
+async def get_stock_quote(
+    symbol: str
+) -> Union[str, TextContent]:
+    """
+    Get current stock quote information.
+    
+    Args:
+        symbol: Stock ticker symbol (e.g., AAPL, MSFT)
+        
+    Returns:
+        TextContent with quote data
+    """
+    try:
+        start_time = time.time()
+        logging.info(f"📈 Fetching stock quote for: {symbol}")
+        
+        ticker = yf.Ticker(symbol)
+        info = ticker.info
+        
+        if not info or (info.get("regularMarketPrice") is None and info.get("currentPrice") is None):
+            # Try to get basic history to validate symbol
+            hist = ticker.history(period="1d")
+            if hist.empty:
+                raise ValueError(f"No data found for symbol: {symbol}")
+            raise ValueError(f"Could not retrieve detailed quote for symbol: {symbol}")
+        
+        # Extract key quote information
+        quote_data = {
+            "symbol": symbol.upper(),
+            "company_name": info.get("shortName", info.get("longName")),
+            "current_price": info.get("regularMarketPrice", info.get("currentPrice")),
+            "previous_close": info.get("previousClose"),
+            "open": info.get("regularMarketOpen", info.get("open")),
+            "day_high": info.get("regularMarketDayHigh", info.get("dayHigh")),
+            "day_low": info.get("regularMarketDayLow", info.get("dayLow")),
+            "volume": info.get("regularMarketVolume", info.get("volume")),
+            "average_volume": info.get("averageVolume"),
+            "market_cap": info.get("marketCap"),
+            "fifty_two_week_high": info.get("fiftyTwoWeekHigh"),
+            "fifty_two_week_low": info.get("fiftyTwoWeekLow"),
+            "currency": info.get("currency"),
+            "exchange": info.get("exchange")
+        }
+        
+        # Filter out None values
+        quote_data = {k: v for k, v in quote_data.items() if v is not None}
+        
+        # Calculate change
+        if quote_data.get("current_price") and quote_data.get("previous_close"):
+            change = quote_data["current_price"] - quote_data["previous_close"]
+            change_pct = (change / quote_data["previous_close"]) * 100
+            quote_data["change"] = round(change, 2)
+            quote_data["change_percent"] = round(change_pct, 2)
+        
+        execution_time = time.time() - start_time
+        
+        metadata = YFinanceMetadata(
+            symbol=symbol.upper(),
+            operation="get_stock_quote",
+            execution_time=execution_time,
+            data_points=len(quote_data),
+            timestamp=datetime.now().isoformat()
+        )
+        
+        logging.info(f"✅ Stock quote: ${quote_data.get('current_price')}")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=quote_data,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Failed to fetch stock quote: {str(e)}"
+        logging.error(f"Stock quote error: {traceback.format_exc()}")
+        
+        metadata = YFinanceMetadata(
+            symbol=symbol.upper(),
+            operation="get_stock_quote",
+            error_type=type(e).__name__,
+            timestamp=datetime.now().isoformat()
+        )
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_historical_data(
+    symbol: str,
+    start: str,
+    end: str,
+    interval: str = "1d",
+    max_rows_preview: int = 10
+) -> Union[str, TextContent]:
+    """
+    Retrieve historical stock data.
+    
+    Args:
+        symbol: Stock ticker symbol
+        start: Start date (YYYY-MM-DD)
+        end: End date (YYYY-MM-DD)
+        interval: Data interval (1d, 1wk, 1mo, etc.)
+        max_rows_preview: Maximum rows to show in preview (0 for all)
+        
+    Returns:
+        TextContent with historical data
+    """
+    try:
+        start_time = time.time()
+        logging.info(f"📈 Fetching historical data for: {symbol}")
+        
+        ticker = yf.Ticker(symbol)
+        hist_df = ticker.history(start=start, end=end, interval=interval)
+        
+        if hist_df.empty:
+            raise ValueError(f"No historical data found for {symbol}")
+        
+        # Convert DataFrame to list of dictionaries
+        hist_df.reset_index(inplace=True)
+        
+        # Ensure date columns are strings
+        if "Date" in hist_df.columns:
+            hist_df["Date"] = hist_df["Date"].astype(str)
+        if "Datetime" in hist_df.columns:
+            hist_df["Datetime"] = hist_df["Datetime"].astype(str)
+        
+        # Clean column names
+        hist_df.columns = hist_df.columns.str.replace(" ", "")
+        
+        historical_data = hist_df.to_dict(orient="records")
+        execution_time = time.time() - start_time
+        
+        # Prepare result with preview
+        result = {
+            "symbol": symbol.upper(),
+            "start_date": start,
+            "end_date": end,
+            "interval": interval,
+            "total_records": len(historical_data),
+            "data": historical_data if max_rows_preview == 0 else historical_data[:max_rows_preview]
+        }
+        
+        metadata = YFinanceMetadata(
+            symbol=symbol.upper(),
+            operation="get_historical_data",
+            execution_time=execution_time,
+            data_points=len(historical_data),
+            timestamp=datetime.now().isoformat()
+        )
+        
+        logging.info(f"✅ Retrieved {len(historical_data)} historical records")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Failed to fetch historical data: {str(e)}"
+        logging.error(f"Historical data error: {traceback.format_exc()}")
+        
+        metadata = YFinanceMetadata(
+            symbol=symbol.upper(),
+            operation="get_historical_data",
+            error_type=type(e).__name__,
+            timestamp=datetime.now().isoformat()
+        )
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_company_info(
+    symbol: str
+) -> Union[str, TextContent]:
+    """
+    Get company information and business details.
+    
+    Args:
+        symbol: Stock ticker symbol
+        
+    Returns:
+        TextContent with company information
+    """
+    try:
+        start_time = time.time()
+        logging.info(f"🏢 Fetching company info for: {symbol}")
+        
+        ticker = yf.Ticker(symbol)
+        info = ticker.info
+        
+        if not info or not info.get("symbol"):
+            raise ValueError(f"No company information found for symbol: {symbol}")
+        
+        # Extract key company information
+        company_data = {
+            "symbol": info.get("symbol"),
+            "short_name": info.get("shortName"),
+            "long_name": info.get("longName"),
+            "sector": info.get("sector"),
+            "industry": info.get("industry"),
+            "full_time_employees": info.get("fullTimeEmployees"),
+            "business_summary": info.get("longBusinessSummary"),
+            "city": info.get("city"),
+            "state": info.get("state"),
+            "country": info.get("country"),
+            "website": info.get("website"),
+            "exchange": info.get("exchange"),
+            "currency": info.get("currency"),
+            "market_cap": info.get("marketCap"),
+            "pe_ratio": info.get("trailingPE"),
+            "forward_pe": info.get("forwardPE"),
+            "dividend_yield": info.get("dividendYield"),
+            "beta": info.get("beta")
+        }
+        
+        # Filter out None values
+        company_data = {k: v for k, v in company_data.items() if v is not None}
+        
+        execution_time = time.time() - start_time
+        
+        metadata = YFinanceMetadata(
+            symbol=symbol.upper(),
+            operation="get_company_info",
+            execution_time=execution_time,
+            data_points=len(company_data),
+            timestamp=datetime.now().isoformat()
+        )
+        
+        logging.info(f"✅ Retrieved company info: {company_data.get('long_name', company_data.get('short_name'))}")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=company_data,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Failed to fetch company info: {str(e)}"
+        logging.error(f"Company info error: {traceback.format_exc()}")
+        
+        metadata = YFinanceMetadata(
+            symbol=symbol.upper(),
+            operation="get_company_info",
+            error_type=type(e).__name__,
+            timestamp=datetime.now().isoformat()
+        )
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+
+
+async def get_financial_statements(
+    symbol: str,
+    statement_type: Literal["income_statement", "balance_sheet", "cash_flow"],
+    period_type: Literal["annual", "quarterly"] = "annual",
+    max_columns_preview: int = 4
+) -> Union[str, TextContent]:
+    """
+    Get financial statements for a company.
+    
+    Args:
+        symbol: Stock ticker symbol
+        statement_type: Type of statement (income_statement, balance_sheet, cash_flow)
+        period_type: Period type (annual or quarterly)
+        max_columns_preview: Maximum periods to show (0 for all)
+        
+    Returns:
+        TextContent with financial statement data
+    """
+    try:
+        start_time = time.time()
+        logging.info(f"📋 Fetching {statement_type} for: {symbol}")
+        
+        ticker = yf.Ticker(symbol)
+        statement_df = None
+        
+        # Get appropriate statement
+        if statement_type == "income_statement":
+            statement_df = ticker.income_stmt if period_type == "annual" else ticker.quarterly_income_stmt
+        elif statement_type == "balance_sheet":
+            statement_df = ticker.balance_sheet if period_type == "annual" else ticker.quarterly_balance_sheet
+        elif statement_type == "cash_flow":
+            statement_df = ticker.cashflow if period_type == "annual" else ticker.quarterly_cashflow
+        else:
+            raise ValueError(f"Invalid statement_type: {statement_type}")
+        
+        if statement_df is None or statement_df.empty:
+            raise ValueError(f"No {period_type} {statement_type} data found for symbol {symbol}")
+        
+        # Process DataFrame
+        statement_df.reset_index(inplace=True)
+        statement_df.rename(columns={"index": "Item"}, inplace=True)
+        
+        # Convert date columns to strings
+        for col in statement_df.columns:
+            if col != "Item":
+                try:
+                    if hasattr(col, "strftime"):
+                        new_col_name = col.strftime("%Y-%m-%d")
+                        statement_df.rename(columns={col: new_col_name}, inplace=True)
+                except Exception:
+                    pass
+        
+        # Limit columns if needed
+        if max_columns_preview > 0 and len(statement_df.columns) > (max_columns_preview + 1):
+            columns_to_keep = ["Item"] + list(statement_df.columns[1:max_columns_preview + 1])
+            statement_df = statement_df[columns_to_keep]
+        
+        statement_data = statement_df.to_dict(orient="records")
+        execution_time = time.time() - start_time
+        
+        result = {
+            "symbol": symbol.upper(),
+            "statement_type": statement_type,
+            "period_type": period_type,
+            "total_line_items": len(statement_data),
+            "periods": len(statement_df.columns) - 1,
+            "data": statement_data
+        }
+        
+        metadata = YFinanceMetadata(
+            symbol=symbol.upper(),
+            operation="get_financial_statements",
+            execution_time=execution_time,
+            data_points=len(statement_data),
+            timestamp=datetime.now().isoformat()
+        )
+        
+        logging.info(f"✅ Retrieved {statement_type}: {len(statement_data)} items")
+        
+        action_response = ActionResponse(
+            success=True,
+            message=result,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )
+        
+    except Exception as e:
+        error_msg = f"Failed to fetch financial statements: {str(e)}"
+        logging.error(f"Financial statements error: {traceback.format_exc()}")
+        
+        metadata = YFinanceMetadata(
+            symbol=symbol.upper(),
+            operation="get_financial_statements",
+            error_type=type(e).__name__,
+            timestamp=datetime.now().isoformat()
+        )
+        
+        action_response = ActionResponse(
+            success=False,
+            message=error_msg,
+            metadata=metadata.model_dump()
+        )
+        
+        return TextContent(
+            type="text",
+            text=json.dumps(action_response.model_dump())
+        )