ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter9/gaia-experience/AWorld/examples/gaia/mcp_collections/documents/models.py

@@ -0,0 +1,24 @@
+from pydantic import BaseModel, Field
+
+
+class DocumentMetadata(BaseModel):
+    """Metadata extracted from document processing."""
+
+    file_name: str = Field(description="Original file name")
+    file_size: int = Field(description="File size in bytes")
+    file_type: str = Field(description="Document file type/extension")
+    absolute_path: str = Field(description="Absolute path to the document file")
+    page_count: int | None = Field(default=None, description="Number of pages in document")
+    processing_time: float = Field(
+        description="Time taken to process the document in seconds", deprecated=True, exclude=True
+    )
+    extracted_images: list[str] = Field(default_factory=list, description="Paths to extracted image files")
+    extracted_media: list[dict[str, str]] = Field(
+        default_factory=list, description="list of extracted media files with type and path"
+    )
+    output_format: str = Field(description="Format of the extracted content")
+    llm_enhanced: bool = Field(default=False, description="Whether LLM enhancement was used", exclude=True)
+    ocr_applied: bool = Field(default=False, description="Whether OCR was applied", exclude=True)
+    extracted_text_file_path: str | None = Field(
+        default=None, description="Absolute path to the extracted text file (if applicable)"
+    )

chapter9/gaia-experience/AWorld/examples/gaia/mcp_collections/documents/mscsv.py

@@ -0,0 +1,376 @@
+import json
+import os
+import time
+import traceback
+from pathlib import Path
+from typing import Any, Literal
+
+import chardet
+import pandas as pd
+from dotenv import load_dotenv
+from pydantic import Field
+from pydantic.fields import FieldInfo
+
+from aworld.logs.util import Color
+from examples.gaia.mcp_collections.base import ActionArguments, ActionCollection, ActionResponse
+from examples.gaia.mcp_collections.documents.models import DocumentMetadata
+
+
+class CSVExtractionCollection(ActionCollection):
+    """MCP service for CSV document content extraction using pandas.
+
+    Supports extraction from CSV files with various encodings and delimiters.
+    Provides LLM-friendly text output with structured metadata and data analysis.
+    """
+
+    def __init__(self, arguments: ActionArguments) -> None:
+        super().__init__(arguments)
+        self._media_output_dir = self.workspace / "extracted_media"
+        self._media_output_dir.mkdir(exist_ok=True)
+
+        self.supported_extensions: set = {".csv", ".tsv", ".txt"}
+
+        self._color_log("CSV Extraction Service initialized", Color.green, "debug")
+        self._color_log(f"Media output directory: {self._media_output_dir}", Color.blue, "debug")
+
+    def _detect_encoding(self, file_path: Path) -> str:
+        """Detect file encoding using chardet.
+
+        Args:
+            file_path: Path to the CSV file
+
+        Returns:
+            Detected encoding string
+        """
+        try:
+            with open(file_path, "rb") as f:
+                raw_data = f.read(10000)  # Read first 10KB for detection
+                result = chardet.detect(raw_data)
+                encoding = result.get("encoding", "utf-8")
+                confidence = result.get("confidence", 0)
+
+                self._color_log(f"Detected encoding: {encoding} (confidence: {confidence:.2f})", Color.blue)
+                return encoding if confidence > 0.7 else "utf-8"
+        except Exception as e:
+            self.logger.warning(f"Encoding detection failed: {e}, using utf-8")
+            return "utf-8"
+
+    def _detect_delimiter(self, file_path: Path, encoding: str) -> str:
+        """Detect CSV delimiter by analyzing the first few lines.
+
+        Args:
+            file_path: Path to the CSV file
+            encoding: File encoding
+
+        Returns:
+            Detected delimiter character
+        """
+        try:
+            with open(file_path, "r", encoding=encoding) as f:
+                sample = f.read(1024)  # Read first 1KB
+
+            # Common delimiters to test
+            delimiters = [",", ";", "\t", "|", ":"]
+            delimiter_counts = {}
+
+            for delimiter in delimiters:
+                count = sample.count(delimiter)
+                if count > 0:
+                    delimiter_counts[delimiter] = count
+
+            if delimiter_counts:
+                detected_delimiter = max(delimiter_counts, key=delimiter_counts.get)
+                self._color_log(f"Detected delimiter: '{detected_delimiter}'", Color.blue)
+                return detected_delimiter
+            else:
+                return ","
+        except Exception as e:
+            self.logger.warning(f"Delimiter detection failed: {e}, using comma")
+            return ","
+
+    def _extract_csv_content(
+        self, file_path: Path, max_rows: int | None = None, encoding: str | None = None, delimiter: str | None = None
+    ) -> dict[str, Any]:
+        """Extract content from CSV file using pandas.
+
+        Args:
+            file_path: Path to the CSV file
+            max_rows: Maximum number of rows to read
+            encoding: File encoding (auto-detected if None)
+            delimiter: CSV delimiter (auto-detected if None)
+
+        Returns:
+            Dictionary containing extracted content and metadata
+        """
+        start_time = time.time()
+
+        # Auto-detect encoding and delimiter if not provided
+        if encoding is None:
+            encoding = self._detect_encoding(file_path)
+        if delimiter is None:
+            delimiter = self._detect_delimiter(file_path, encoding)
+
+        try:
+            # Read CSV with pandas
+            df = pd.read_csv(file_path, encoding=encoding, delimiter=delimiter, nrows=max_rows, low_memory=False)
+
+            # Get full file info for metadata
+            full_df_info = pd.read_csv(
+                file_path,
+                encoding=encoding,
+                delimiter=delimiter,
+                nrows=0,  # Just get headers and shape info
+            )
+
+            # Count total rows efficiently
+            total_rows = sum(1 for _ in open(file_path, "r", encoding=encoding)) - 1  # Subtract header
+
+            processing_time = time.time() - start_time
+
+            return {
+                "dataframe": df,
+                "total_rows": total_rows,
+                "total_columns": len(full_df_info.columns),
+                "columns": list(df.columns),
+                "encoding": encoding,
+                "delimiter": delimiter,
+                "processing_time": processing_time,
+                "data_types": df.dtypes.to_dict(),
+                "memory_usage": df.memory_usage(deep=True).sum(),
+            }
+
+        except Exception as e:
+            self.logger.error(f"Failed to read CSV file: {e}")
+            raise
+
+    def _format_content_for_llm(self, df: pd.DataFrame, output_format: str, include_stats: bool = True) -> str:
+        """Format extracted CSV content to be LLM-friendly.
+
+        Args:
+            df: Pandas DataFrame with CSV data
+            output_format: Desired output format
+            include_stats: Whether to include statistical summary
+
+        Returns:
+            Formatted content string
+        """
+        if output_format.lower() == "markdown":
+            # Convert to markdown table
+            content = df.to_markdown(index=False, tablefmt="github")
+
+            if include_stats:
+                # Add statistical summary
+                stats_content = "\n\n## Data Summary\n\n"
+                stats_content += f"- **Rows**: {len(df)}\n"
+                stats_content += f"- **Columns**: {len(df.columns)}\n"
+                stats_content += f"- **Column Names**: {', '.join(df.columns)}\n\n"
+
+                # Add data types info
+                stats_content += "### Column Data Types\n\n"
+                for col, dtype in df.dtypes.items():
+                    stats_content += f"- **{col}**: {dtype}\n"
+
+                # Add basic statistics for numeric columns
+                numeric_cols = df.select_dtypes(include=["number"]).columns
+                if len(numeric_cols) > 0:
+                    stats_content += "\n### Numeric Column Statistics\n\n"
+                    stats_df = df[numeric_cols].describe()
+                    stats_content += stats_df.to_markdown(tablefmt="github")
+
+                content += stats_content
+
+        elif output_format.lower() == "json":
+            # Convert to JSON with metadata
+            data_dict = {
+                "data": df.to_dict(orient="records"),
+                "metadata": {
+                    "rows": len(df),
+                    "columns": len(df.columns),
+                    "column_names": list(df.columns),
+                    "data_types": {col: str(dtype) for col, dtype in df.dtypes.items()},
+                },
+            }
+            if include_stats:
+                numeric_cols = df.select_dtypes(include=["number"]).columns
+                if len(numeric_cols) > 0:
+                    data_dict["statistics"] = df[numeric_cols].describe().to_dict()
+
+            content = json.dumps(data_dict, indent=2, default=str)
+
+        elif output_format.lower() == "html":
+            # Convert to HTML table
+            content = df.to_html(index=False, classes="table table-striped")
+
+        else:
+            # Plain text format
+            content = df.to_string(index=False)
+
+        return content
+
+    def mcp_extract_csv_content(
+        self,
+        file_path: str = Field(description="Path to the CSV document file to extract content from"),
+        output_format: Literal["markdown", "json", "html", "text"] = Field(
+            default="markdown", description="Output format: 'markdown', 'json', 'html', or 'text'"
+        ),
+        max_rows: int | None = Field(default=None, description="Maximum number of rows to read (None for all rows)"),
+        include_statistics: bool = Field(default=True, description="Whether to include statistical summary in output"),
+        generate_visualizations: bool = Field(
+            default=False, description="Whether to generate and save data visualizations"
+        ),
+        encoding: str | None = Field(default=None, description="File encoding (auto-detected if None)"),
+        delimiter: str | None = Field(default=None, description="CSV delimiter (auto-detected if None)"),
+    ) -> ActionResponse:
+        """Extract content from CSV documents using pandas.
+
+        This tool provides comprehensive CSV document content extraction with support for:
+        - CSV, TSV, and delimited text files
+        - Automatic encoding and delimiter detection
+        - Statistical analysis and data profiling
+        - Multiple output formats (Markdown, JSON, HTML, Text)
+        - Optional data visualizations
+        - Memory-efficient processing for large files
+
+        Args:
+            file_path: Path to the CSV file
+            output_format: Desired output format
+            max_rows: Maximum rows to process (None for all)
+            include_statistics: Include statistical summary
+            generate_visualizations: Generate data visualizations
+            encoding: File encoding (auto-detected if None)
+            delimiter: CSV delimiter (auto-detected if None)
+
+        Returns:
+            ActionResponse with extracted content, metadata, and optional visualizations
+        """
+        try:
+            # Handle FieldInfo objects from pydantic
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+            if isinstance(output_format, FieldInfo):
+                output_format = output_format.default
+            if isinstance(max_rows, FieldInfo):
+                max_rows = max_rows.default
+            if isinstance(include_statistics, FieldInfo):
+                include_statistics = include_statistics.default
+            if isinstance(generate_visualizations, FieldInfo):
+                generate_visualizations = generate_visualizations.default
+            if isinstance(encoding, FieldInfo):
+                encoding = encoding.default
+            if isinstance(delimiter, FieldInfo):
+                delimiter = delimiter.default
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Processing CSV file: {file_path.name}", Color.cyan)
+
+            # Extract CSV content
+            extraction_result = self._extract_csv_content(
+                file_path, max_rows=max_rows, encoding=encoding, delimiter=delimiter
+            )
+
+            df: pd.DataFrame = extraction_result["dataframe"]
+
+            # Format content for LLM consumption
+            formatted_content = self._format_content_for_llm(df, output_format, include_stats=include_statistics)
+
+            # Prepare metadata
+            file_stats = file_path.stat()
+            document_metadata = DocumentMetadata(
+                file_name=file_path.name,
+                file_size=file_stats.st_size,
+                file_type=file_path.suffix.lower(),
+                absolute_path=str(file_path.absolute()),
+                page_count=None,  # Not applicable for CSV
+                processing_time=extraction_result["processing_time"],
+                extracted_images=[],  # CSV files don't contain images
+                extracted_media=None,
+                output_format=output_format,
+                llm_enhanced=False,
+                ocr_applied=False,
+            )
+
+            # Add CSV-specific metadata
+            csv_metadata = {
+                "total_rows": extraction_result["total_rows"],
+                "total_columns": extraction_result["total_columns"],
+                "rows_processed": len(df),
+                "columns_processed": len(df.columns),
+                "column_names": extraction_result["columns"],
+                "data_types": {k: str(v) for k, v in extraction_result["data_types"].items()},
+                "encoding": extraction_result["encoding"],
+                "delimiter": extraction_result["delimiter"],
+                "memory_usage_bytes": int(extraction_result["memory_usage"]),
+            }
+
+            # Merge metadata
+            final_metadata = {**document_metadata.model_dump(), **csv_metadata}
+
+            self._color_log(
+                f"Successfully extracted CSV content from {file_path.name} "
+                f"({extraction_result['total_rows']} rows, {extraction_result['total_columns']} columns",
+                Color.green,
+            )
+
+            return ActionResponse(success=True, message=formatted_content, metadata=final_metadata)
+
+        except FileNotFoundError as e:
+            self.logger.error(f"File not found: {str(e)}")
+            return ActionResponse(
+                success=False, message=f"File not found: {str(e)}", metadata={"error_type": "file_not_found"}
+            )
+        except ValueError as e:
+            self.logger.error(f"Invalid input: {str(e)}")
+            return ActionResponse(
+                success=False,
+                message=f"Invalid input: {str(e)}",
+                metadata={"error_type": "invalid_input"},
+            )
+        except Exception as e:
+            self.logger.error(f"CSV extraction failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"CSV extraction failed: {str(e)}",
+                metadata={"error_type": "extraction_error"},
+            )
+
+    def mcp_list_supported_formats(self) -> ActionResponse:
+        """List all supported CSV formats for extraction.
+
+        Returns:
+            ActionResponse with list of supported file formats and their descriptions
+        """
+        supported_formats = {
+            "CSV": "Comma-Separated Values files (.csv)",
+            "TSV": "Tab-Separated Values files (.tsv)",
+            "TXT": "Delimited text files (.txt)",
+        }
+
+        format_list = "\n".join(
+            [f"**{format_name}**: {description}" for format_name, description in supported_formats.items()]
+        )
+
+        return ActionResponse(
+            success=True,
+            message=f"Supported CSV formats:\n\n{format_list}",
+            metadata={"supported_formats": list(supported_formats.keys()), "total_formats": len(supported_formats)},
+        )
+
+
+# Example usage and entry point
+if __name__ == "__main__":
+    load_dotenv()
+
+    # Default arguments for testing
+    args = ActionArguments(
+        name="csv_extraction_service",
+        transport="stdio",
+        workspace=os.getenv("AWORLD_WORKSPACE", "~"),
+    )
+
+    # Initialize and run the CSV extraction service
+    try:
+        service = CSVExtractionCollection(args)
+        service.run()
+    except Exception as e:
+        print(f"An error occurred: {e}: {traceback.format_exc()}")

chapter9/gaia-experience/AWorld/examples/gaia/mcp_collections/documents/msdocx.py

@@ -0,0 +1,617 @@
+import json
+import os
+import time
+import traceback
+import zipfile
+from pathlib import Path
+from typing import Any, Literal
+
+from docx import Document
+from docx.document import Document as DocumentType
+from dotenv import load_dotenv
+from pydantic import Field
+from pydantic.fields import FieldInfo
+
+from aworld.logs.util import Color
+from examples.gaia.mcp_collections.base import ActionArguments, ActionCollection, ActionResponse
+from examples.gaia.mcp_collections.documents.models import DocumentMetadata
+
+
+class DOCXExtractionCollection(ActionCollection):
+    """MCP service for DOCX/DOC document content extraction using python-docx.
+
+    Supports extraction from DOCX and DOC files with comprehensive content parsing.
+    Provides LLM-friendly text output with structured metadata and media file handling.
+    """
+
+    def __init__(self, arguments: ActionArguments) -> None:
+        super().__init__(arguments)
+        self._media_output_dir = self.workspace / "extracted_media"
+        self._media_output_dir.mkdir(exist_ok=True)
+
+        self.supported_extensions = {".docx", ".doc"}
+
+        self._color_log("DOCX Extraction Service initialized", Color.green, "debug")
+        self._color_log(f"Media output directory: {self._media_output_dir}", Color.blue, "debug")
+
+    def _extract_images_from_docx(self, file_path: Path, file_stem: str) -> list[dict[str, str]]:
+        """Extract embedded images from DOCX file.
+
+        Args:
+            file_path: Path to the DOCX file
+            file_stem: Base name for saving files
+
+        Returns:
+            List of dictionaries containing image file paths and metadata
+        """
+        saved_media = []
+
+        try:
+            # DOCX files are ZIP archives
+            with zipfile.ZipFile(file_path, "r") as docx_zip:
+                # Look for media files in the word/media/ directory
+                media_files = [f for f in docx_zip.namelist() if f.startswith("word/media/")]
+
+                for idx, media_file in enumerate(media_files):
+                    try:
+                        # Extract file extension and create appropriate filename
+                        original_name = Path(media_file).name
+                        file_extension = Path(media_file).suffix
+
+                        # Generate unique filename
+                        media_filename = f"{file_stem}_media_{idx}_{original_name}"
+                        media_path = self._media_output_dir / media_filename
+
+                        # Extract and save the media file
+                        with docx_zip.open(media_file) as source:
+                            with open(media_path, "wb") as target:
+                                target.write(source.read())
+
+                        # Determine media type based on extension
+                        media_type = "image"
+                        if file_extension.lower() in [".mp3", ".wav", ".m4a", ".ogg"]:
+                            media_type = "audio"
+                        elif file_extension.lower() in [".mp4", ".avi", ".mov", ".wmv"]:
+                            media_type = "video"
+                        elif file_extension.lower() in [".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff"]:
+                            media_type = "image"
+                        else:
+                            media_type = "other"
+
+                        saved_media.append(
+                            {
+                                "type": media_type,
+                                "path": str(media_path),
+                                "filename": media_filename,
+                                "original_name": original_name,
+                                "size_bytes": media_path.stat().st_size,
+                            }
+                        )
+
+                        self._color_log(f"Extracted {media_type}: {media_filename}", Color.blue)
+
+                    except Exception as e:
+                        self.logger.error(f"Failed to extract media file {media_file}: {e}")
+
+        except Exception as e:
+            self.logger.warning(f"Could not extract media from DOCX: {e}")
+
+        return saved_media
+
+    def _extract_document_structure(self, doc: DocumentType) -> dict[str, Any]:
+        """Extract document structure and metadata.
+
+        Args:
+            doc: python-docx Document object
+
+        Returns:
+            Dictionary containing document structure information
+        """
+        structure = {
+            "paragraphs_count": len(doc.paragraphs),
+            "tables_count": len(doc.tables),
+            "sections_count": len(doc.sections),
+            "styles_used": [],
+            "has_headers_footers": False,
+            "page_count": None,  # Not directly available in python-docx
+        }
+
+        # Collect unique styles used in the document
+        styles_used = set()
+        for paragraph in doc.paragraphs:
+            if paragraph.style and paragraph.style.name:
+                styles_used.add(paragraph.style.name)
+
+        structure["styles_used"] = list(styles_used)
+
+        # Check for headers and footers
+        for section in doc.sections:
+            if (section.header.paragraphs and any(p.text.strip() for p in section.header.paragraphs)) or (
+                section.footer.paragraphs and any(p.text.strip() for p in section.footer.paragraphs)
+            ):
+                structure["has_headers_footers"] = True
+                break
+
+        return structure
+
+    def _extract_tables_content(self, doc: DocumentType) -> list[dict[str, Any]]:
+        """Extract content from all tables in the document.
+
+        Args:
+            doc: python-docx Document object
+
+        Returns:
+            List of dictionaries containing table data
+        """
+        tables_data = []
+
+        for table_idx, table in enumerate(doc.tables):
+            table_data = {
+                "table_index": table_idx,
+                "rows_count": len(table.rows),
+                "columns_count": len(table.columns) if table.rows else 0,
+                "data": [],
+            }
+
+            # Extract table content
+            for _, row in enumerate(table.rows):
+                row_data = []
+                for cell in row.cells:
+                    cell_text = cell.text.strip()
+                    row_data.append(cell_text)
+                table_data["data"].append(row_data)
+
+            tables_data.append(table_data)
+
+        return tables_data
+
+    def _convert_doc_to_docx(self, doc_path: Path) -> Path:
+        """Convert .doc file to .docx using LibreOffice.
+
+        Args:
+            doc_path: Path to the .doc file
+
+        Returns:
+            Path to the converted .docx file
+        """
+        import subprocess
+
+        output_dir = self._media_output_dir
+        docx_path = output_dir / f"{doc_path.stem}.docx"
+
+        # Check if already converted
+        if docx_path.exists():
+            self._color_log(f"Using existing converted file: {docx_path.name}", Color.blue)
+            return docx_path
+
+        self._color_log(f"Converting .doc to .docx: {doc_path.name}", Color.yellow)
+
+        cmd = ["libreoffice", "--headless", "--convert-to", "docx", "--outdir", str(output_dir), str(doc_path)]
+
+        try:
+            result = subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=60)  # pylint: disable=W0612
+            if docx_path.exists():
+                self._color_log(f"Conversion successful: {docx_path.name}", Color.green)
+                return docx_path
+            else:
+                raise RuntimeError("Conversion completed but output file not found")
+
+        except subprocess.CalledProcessError as e:
+            self.logger.error(f"LibreOffice conversion failed: {e.stderr}")
+            raise RuntimeError(f"Failed to convert .doc to .docx: {e.stderr}") from e
+        except subprocess.TimeoutExpired as e:
+            self.logger.error("LibreOffice conversion timed out")
+            raise RuntimeError("Conversion timed out after 60 seconds") from e
+
+    # Modify the _extract_content_from_docx method to handle .doc files
+    def _extract_content_from_docx(
+        self, file_path: Path, extract_tables: bool = True, extract_headers_footers: bool = True
+    ) -> dict[str, Any]:
+        """Extract content from DOCX file using python-docx.
+
+        Args:
+            file_path: Path to the DOCX/DOC file
+            extract_tables: Whether to extract table content
+            extract_headers_footers: Whether to extract headers and footers
+
+        Returns:
+            Dictionary containing extracted content and metadata
+        """
+        start_time = time.time()
+
+        # Convert .doc to .docx if needed
+        if file_path.suffix.lower() == ".doc":
+            file_path = self._convert_doc_to_docx(file_path)
+
+        try:
+            # Load the document
+            doc = Document(str(file_path))
+
+            # Extract main text content
+            paragraphs = []
+            for paragraph in doc.paragraphs:
+                if paragraph.text.strip():  # Skip empty paragraphs
+                    para_data = {"text": paragraph.text, "style": paragraph.style.name if paragraph.style else "Normal"}
+                    paragraphs.append(para_data)
+
+            # Extract document structure
+            structure = self._extract_document_structure(doc)
+
+            # Extract tables if requested
+            tables = []
+            if extract_tables:
+                tables = self._extract_tables_content(doc)
+
+            # Extract headers and footers if requested
+            headers_footers = []
+            if extract_headers_footers:
+                for section_idx, section in enumerate(doc.sections):
+                    # Extract header content
+                    header_text = []
+                    for para in section.header.paragraphs:
+                        if para.text.strip():
+                            header_text.append(para.text)
+
+                    # Extract footer content
+                    footer_text = []
+                    for para in section.footer.paragraphs:
+                        if para.text.strip():
+                            footer_text.append(para.text)
+
+                    if header_text or footer_text:
+                        headers_footers.append(
+                            {"section_index": section_idx, "header": header_text, "footer": footer_text}
+                        )
+
+            processing_time = time.time() - start_time
+
+            return {
+                "paragraphs": paragraphs,
+                "tables": tables,
+                "headers_footers": headers_footers,
+                "structure": structure,
+                "processing_time": processing_time,
+                "word_count": sum(len(p["text"].split()) for p in paragraphs),
+                "character_count": sum(len(p["text"]) for p in paragraphs),
+            }
+
+        except Exception as e:
+            self.logger.error(f"Failed to extract content from DOCX: {e}")
+            raise
+
+    def _format_content_for_llm(
+        self, extraction_result: dict[str, Any], output_format: str, include_structure: bool = True
+    ) -> str:
+        """Format extracted DOCX content to be LLM-friendly.
+
+        Args:
+            extraction_result: Dictionary containing extracted content
+            output_format: Desired output format
+            include_structure: Whether to include document structure information
+
+        Returns:
+            Formatted content string
+        """
+        if output_format.lower() == "markdown":
+            content_parts = []
+
+            # Add document structure info if requested
+            if include_structure:
+                structure = extraction_result["structure"]
+                content_parts.append("# Document Structure\n")
+                content_parts.append(f"- **Paragraphs**: {structure['paragraphs_count']}\n")
+                content_parts.append(f"- **Tables**: {structure['tables_count']}\n")
+                content_parts.append(f"- **Sections**: {structure['sections_count']}\n")
+                content_parts.append(f"- **Word Count**: {extraction_result['word_count']}\n")
+                content_parts.append(f"- **Character Count**: {extraction_result['character_count']}\n")
+                if structure["styles_used"]:
+                    content_parts.append(f"- **Styles Used**: {', '.join(structure['styles_used'])}\n")
+                content_parts.append("\n---\n\n")
+
+            # Add main content
+            content_parts.append("# Document Content\n\n")
+
+            # Add paragraphs
+            for para in extraction_result["paragraphs"]:
+                # Format based on style
+                text = para["text"]
+                style = para["style"]
+
+                if "Heading" in style:
+                    # Convert heading styles to markdown headers
+                    if "Heading 1" in style:
+                        content_parts.append(f"# {text}\n\n")
+                    elif "Heading 2" in style:
+                        content_parts.append(f"## {text}\n\n")
+                    elif "Heading 3" in style:
+                        content_parts.append(f"### {text}\n\n")
+                    else:
+                        content_parts.append(f"#### {text}\n\n")
+                else:
+                    content_parts.append(f"{text}\n\n")
+
+            # Add tables
+            if extraction_result["tables"]:
+                content_parts.append("\n## Tables\n\n")
+                for table_idx, table in enumerate(extraction_result["tables"]):
+                    content_parts.append(f"### Table {table_idx + 1}\n\n")
+
+                    if table["data"]:
+                        # Create markdown table
+                        headers = table["data"][0] if table["data"] else []
+                        if headers:
+                            content_parts.append("| " + " | ".join(headers) + " |\n")
+                            content_parts.append("|" + "---|" * len(headers) + "\n")
+
+                            for row in table["data"][1:]:
+                                content_parts.append("| " + " | ".join(row) + " |\n")
+                        content_parts.append("\n")
+
+            # Add headers and footers
+            if extraction_result["headers_footers"]:
+                content_parts.append("\n## Headers and Footers\n\n")
+                for hf in extraction_result["headers_footers"]:
+                    if hf["header"]:
+                        content_parts.append(f"**Header (Section {hf['section_index'] + 1}):**\n")
+                        for header_line in hf["header"]:
+                            content_parts.append(f"{header_line}\n")
+                        content_parts.append("\n")
+
+                    if hf["footer"]:
+                        content_parts.append(f"**Footer (Section {hf['section_index'] + 1}):**\n")
+                        for footer_line in hf["footer"]:
+                            content_parts.append(f"{footer_line}\n")
+                        content_parts.append("\n")
+
+            return "".join(content_parts)
+
+        elif output_format.lower() == "json":
+            # Return structured JSON
+            return json.dumps(extraction_result, indent=2, ensure_ascii=False)
+
+        elif output_format.lower() == "html":
+            # Convert to HTML
+            html_parts = ["<html><body>"]
+
+            if include_structure:
+                html_parts.append("<h1>Document Structure</h1>")
+                structure = extraction_result["structure"]
+                html_parts.append(f"<p><strong>Paragraphs:</strong> {structure['paragraphs_count']}</p>")
+                html_parts.append(f"<p><strong>Tables:</strong> {structure['tables_count']}</p>")
+                html_parts.append(f"<p><strong>Word Count:</strong> {extraction_result['word_count']}</p>")
+                html_parts.append("<hr>")
+
+            html_parts.append("<h1>Document Content</h1>")
+
+            # Add paragraphs
+            for para in extraction_result["paragraphs"]:
+                text = para["text"].replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+                style = para["style"]
+
+                if "Heading" in style:
+                    if "Heading 1" in style:
+                        html_parts.append(f"<h1>{text}</h1>")
+                    elif "Heading 2" in style:
+                        html_parts.append(f"<h2>{text}</h2>")
+                    elif "Heading 3" in style:
+                        html_parts.append(f"<h3>{text}</h3>")
+                    else:
+                        html_parts.append(f"<h4>{text}</h4>")
+                else:
+                    html_parts.append(f"<p>{text}</p>")
+
+            # Add tables
+            if extraction_result["tables"]:
+                html_parts.append("<h2>Tables</h2>")
+                for table_idx, table in enumerate(extraction_result["tables"]):
+                    html_parts.append(f"<h3>Table {table_idx + 1}</h3>")
+                    html_parts.append("<table border='1'>")
+
+                    for row_idx, row in enumerate(table["data"]):
+                        html_parts.append("<tr>")
+                        tag = "th" if row_idx == 0 else "td"
+                        for cell in row:
+                            cell_text = cell.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+                            html_parts.append(f"<{tag}>{cell_text}</{tag}>")
+                        html_parts.append("</tr>")
+
+                    html_parts.append("</table>")
+
+            html_parts.append("</body></html>")
+            return "".join(html_parts)
+
+        else:
+            # Plain text format
+            text_parts = []
+
+            # Add main content
+            for para in extraction_result["paragraphs"]:
+                text_parts.append(para["text"])
+
+            # Add tables as plain text
+            if extraction_result["tables"]:
+                text_parts.append("\n\n=== TABLES ===\n")
+                for table_idx, table in enumerate(extraction_result["tables"]):
+                    text_parts.append(f"\nTable {table_idx + 1}:\n")
+                    for row in table["data"]:
+                        text_parts.append("\t".join(row) + "\n")
+
+            return "\n\n".join(text_parts)
+
+    def mcp_extract_docx_content(
+        self,
+        file_path: str = Field(description="Path to the DOCX/DOC document file to extract content from"),
+        output_format: Literal["markdown", "json", "html", "text"] = Field(
+            default="markdown", description="Output format: 'markdown', 'json', 'html', or 'text'"
+        ),
+        extract_images: bool = Field(default=True, description="Whether to extract and save embedded images and media"),
+        extract_tables: bool = Field(default=True, description="Whether to extract table content"),
+        extract_headers_footers: bool = Field(default=True, description="Whether to extract headers and footers"),
+        include_structure: bool = Field(
+            default=True, description="Whether to include document structure information in output"
+        ),
+    ) -> ActionResponse:
+        """Extract content from DOCX/DOC documents using python-docx.
+
+        This tool provides comprehensive DOCX/DOC document content extraction with support for:
+        - DOCX and DOC files
+        - Text extraction with style preservation
+        - Table extraction and formatting
+        - Headers and footers extraction
+        - Embedded media extraction (images, audio, etc.)
+        - Multiple output formats (Markdown, JSON, HTML, Text)
+        - Document structure analysis
+
+        Args:
+            file_path: Path to the DOCX/DOC file
+            output_format: Desired output format
+            extract_images: Extract embedded media files
+            extract_tables: Extract table content
+            extract_headers_footers: Extract headers and footers
+            include_structure: Include document structure info
+
+        Returns:
+            ActionResponse with extracted content, metadata, and media file paths
+        """
+        try:
+            # Handle FieldInfo objects from pydantic
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+            if isinstance(output_format, FieldInfo):
+                output_format = output_format.default
+            if isinstance(extract_images, FieldInfo):
+                extract_images = extract_images.default
+            if isinstance(extract_tables, FieldInfo):
+                extract_tables = extract_tables.default
+            if isinstance(extract_headers_footers, FieldInfo):
+                extract_headers_footers = extract_headers_footers.default
+            if isinstance(include_structure, FieldInfo):
+                include_structure = include_structure.default
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Processing DOCX document: {file_path.name}", Color.cyan)
+
+            # Extract embedded media if requested
+            saved_media = []
+            if extract_images and file_path.suffix.lower() == ".docx":
+                saved_media = self._extract_images_from_docx(file_path, file_path.stem)
+
+            # Extract document content
+            extraction_result = self._extract_content_from_docx(
+                file_path, extract_tables=extract_tables, extract_headers_footers=extract_headers_footers
+            )
+
+            # Format content for LLM consumption
+            formatted_content = self._format_content_for_llm(
+                extraction_result, output_format, include_structure=include_structure
+            )
+
+            # Prepare metadata
+            file_stats = file_path.stat()
+            document_metadata = DocumentMetadata(
+                file_name=file_path.name,
+                file_size=file_stats.st_size,
+                file_type=file_path.suffix.lower(),
+                absolute_path=str(file_path.absolute()),
+                page_count=None,  # Not directly available for DOCX
+                processing_time=extraction_result["processing_time"],
+                extracted_images=[media["path"] for media in saved_media if media["type"] == "image"],
+                extracted_media=saved_media,
+                output_format=output_format,
+                llm_enhanced=False,
+                ocr_applied=False,
+            )
+
+            # Add DOCX-specific metadata
+            docx_metadata = {
+                "paragraphs_count": extraction_result["structure"]["paragraphs_count"],
+                "tables_count": extraction_result["structure"]["tables_count"],
+                "sections_count": extraction_result["structure"]["sections_count"],
+                "word_count": extraction_result["word_count"],
+                "character_count": extraction_result["character_count"],
+                "styles_used": extraction_result["structure"]["styles_used"],
+                "has_headers_footers": extraction_result["structure"]["has_headers_footers"],
+                "has_embedded_media": len(saved_media) > 0,
+                "media_files_count": len(saved_media),
+            }
+
+            # Merge metadata
+            final_metadata = {**document_metadata.model_dump(), **docx_metadata}
+
+            self._color_log(
+                f"Successfully extracted DOCX content from {file_path.name} "
+                f"({extraction_result['word_count']} words, {extraction_result['structure']['tables_count']} tables, "
+                f"{len(saved_media)} media files)",
+                Color.green,
+            )
+
+            return ActionResponse(success=True, message=formatted_content, metadata=final_metadata)
+
+        except FileNotFoundError as e:
+            self.logger.error(f"File not found: {str(e)}")
+            return ActionResponse(
+                success=False, message=f"File not found: {str(e)}", metadata={"error_type": "file_not_found"}
+            )
+        except ValueError as e:
+            self.logger.error(f"Invalid input: {str(e)}")
+            return ActionResponse(
+                success=False,
+                message=f"Invalid input: {str(e)}",
+                metadata={"error_type": "invalid_input"},
+            )
+        except ImportError as e:
+            self.logger.error(f"Missing dependency: {str(e)}")
+            return ActionResponse(
+                success=False,
+                message=f"Missing dependency: {str(e)}. Please install python-docx: pip install python-docx",
+                metadata={"error_type": "missing_dependency"},
+            )
+        except Exception as e:
+            self.logger.error(f"DOCX extraction failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"DOCX extraction failed: {str(e)}",
+                metadata={"error_type": "extraction_error"},
+            )
+
+    def mcp_list_supported_formats(self) -> ActionResponse:
+        """List all supported document formats for extraction.
+
+        Returns:
+            ActionResponse with list of supported file formats and their descriptions
+        """
+        supported_formats = {
+            "DOCX": "Microsoft Word Open XML Document (.docx)",
+            "DOC": "Microsoft Word Document (.doc) - limited support",
+        }
+
+        format_list = "\n".join(
+            [f"**{format_name}**: {description}" for format_name, description in supported_formats.items()]
+        )
+
+        return ActionResponse(
+            success=True,
+            message=f"Supported document formats:\n\n{format_list}\n\n"
+            "**Note**: DOC format support is limited. For best results, convert to DOCX format.",
+            metadata={"supported_formats": list(supported_formats.keys()), "total_formats": len(supported_formats)},
+        )
+
+
+# Example usage and entry point
+if __name__ == "__main__":
+    load_dotenv()
+
+    # Default arguments for testing
+    args = ActionArguments(
+        name="docx_extraction_service",
+        transport="stdio",
+        workspace=os.getenv("AWORLD_WORKSPACE", "~"),
+    )
+
+    # Initialize and run the DOCX extraction service
+    try:
+        service = DOCXExtractionCollection(args)
+        service.run()
+    except Exception as e:
+        print(f"An error occurred: {e}: {traceback.format_exc()}")

chapter9/gaia-experience/AWorld/examples/gaia/mcp_collections/documents/mspptx.py

@@ -0,0 +1,545 @@
+import json
+import os
+import time
+import traceback
+import zipfile
+from pathlib import Path
+from typing import Any, Literal
+
+from dotenv import load_dotenv
+from pptx import Presentation
+from pptx.presentation import Presentation as PresentationType
+from pydantic import Field
+from pydantic.fields import FieldInfo
+
+from aworld.logs.util import Color
+from examples.gaia.mcp_collections.base import ActionArguments, ActionCollection, ActionResponse
+from examples.gaia.mcp_collections.documents.models import DocumentMetadata
+
+
+class PPTXExtractionCollection(ActionCollection):
+    """MCP service for PPTX/PPT presentation content extraction using python-pptx.
+
+    Supports extraction from PPTX and PPT files with comprehensive content parsing.
+    Provides LLM-friendly text output with structured metadata and media file handling.
+    """
+
+    def __init__(self, arguments: ActionArguments) -> None:
+        super().__init__(arguments)
+        self._media_output_dir = self.workspace / "extracted_media"
+        self._media_output_dir.mkdir(exist_ok=True)
+
+        self.supported_extensions = {".pptx", ".ppt"}
+
+        self._color_log("PPTX Extraction Service initialized", Color.green, "debug")
+        self._color_log(f"Media output directory: {self._media_output_dir}", Color.blue, "debug")
+
+    def _extract_images_from_pptx(self, file_path: Path, file_stem: str) -> list[dict[str, str]]:
+        """Extract embedded images from PPTX file.
+
+        Args:
+            file_path: Path to the PPTX file
+            file_stem: Base name for saving files
+
+        Returns:
+            List of dictionaries containing image file paths and metadata
+        """
+        saved_media = []
+
+        try:
+            # PPTX files are ZIP archives
+            with zipfile.ZipFile(file_path, "r") as zip_file:
+                # Find media files in the archive
+                media_files = [f for f in zip_file.namelist() if f.startswith("ppt/media/")]
+
+                for idx, media_file in enumerate(media_files):
+                    try:
+                        # Extract file extension
+                        original_ext = Path(media_file).suffix
+                        if not original_ext:
+                            original_ext = ".png"  # Default extension
+
+                        # Generate unique filename
+                        media_filename = f"{file_stem}_media_{idx}{original_ext}"
+                        media_path = self._media_output_dir / media_filename
+
+                        # Extract and save media file
+                        with zip_file.open(media_file) as source:
+                            with open(media_path, "wb") as target:
+                                target.write(source.read())
+
+                        # Determine media type
+                        media_type = (
+                            "image" if original_ext.lower() in {".png", ".jpg", ".jpeg", ".gif", ".bmp"} else "media"
+                        )
+
+                        saved_media.append(
+                            {
+                                "type": media_type,
+                                "path": str(media_path),
+                                "filename": media_filename,
+                                "original_path": media_file,
+                            }
+                        )
+
+                        self._color_log(f"Saved media: {media_filename}", Color.blue)
+
+                    except Exception as e:
+                        self.logger.error(f"Failed to extract media {media_file}: {e}")
+
+        except Exception as e:
+            self.logger.error(f"Failed to extract media from PPTX: {e}")
+
+        return saved_media
+
+    def _extract_slide_structure(self, presentation: PresentationType) -> dict[str, Any]:
+        """Extract presentation structure information.
+
+        Args:
+            presentation: python-pptx Presentation object
+
+        Returns:
+            Dictionary containing structure metadata
+        """
+        structure = {
+            "slide_count": len(presentation.slides),
+            "slide_layouts": [],
+            "has_notes": False,
+            "has_comments": False,
+            "slide_sizes": None,
+        }
+
+        # Get slide size information
+        if hasattr(presentation.slide_width, "inches") and hasattr(presentation.slide_height, "inches"):
+            structure["slide_sizes"] = {
+                "width_inches": presentation.slide_width.inches,
+                "height_inches": presentation.slide_height.inches,
+            }
+
+        # Analyze slide layouts and content
+        for slide_idx, slide in enumerate(presentation.slides):
+            layout_info = {
+                "slide_index": slide_idx,
+                "layout_name": slide.slide_layout.name if hasattr(slide.slide_layout, "name") else "Unknown",
+                "shape_count": len(slide.shapes),
+                "has_title": False,
+                "has_content": False,
+            }
+
+            # Check for title and content
+            for shape in slide.shapes:
+                if hasattr(shape, "text") and shape.text.strip():
+                    if (
+                        "title" in str(shape.placeholder_format.type).lower()
+                        if hasattr(shape, "placeholder_format")
+                        else False
+                    ):
+                        layout_info["has_title"] = True
+                    else:
+                        layout_info["has_content"] = True
+
+            # Check for notes
+            if hasattr(slide, "notes_slide") and slide.notes_slide:
+                if hasattr(slide.notes_slide, "notes_text_frame") and slide.notes_slide.notes_text_frame.text.strip():
+                    structure["has_notes"] = True
+
+            structure["slide_layouts"].append(layout_info)
+
+        return structure
+
+    def _extract_content_from_pptx(self, file_path: Path, extract_notes: bool = True) -> dict[str, Any]:
+        """Extract content from PPTX file using python-pptx.
+
+        Args:
+            file_path: Path to the PPTX file
+            extract_notes: Whether to extract speaker notes
+
+        Returns:
+            Dictionary containing extracted content and metadata
+        """
+        start_time = time.time()
+
+        try:
+            # Load the presentation
+            presentation = Presentation(str(file_path))
+
+            # Extract slide content
+            slides_content = []
+            total_text_length = 0
+
+            for slide_idx, slide in enumerate(presentation.slides):
+                slide_data = {
+                    "slide_number": slide_idx + 1,
+                    "title": "",
+                    "content": [],
+                    "notes": "",
+                    "shapes_count": len(slide.shapes),
+                }
+
+                # Extract text from all shapes
+                for shape in slide.shapes:
+                    if hasattr(shape, "text") and shape.text.strip():
+                        text_content = shape.text.strip()
+
+                        # Try to identify if this is a title
+                        if (
+                            hasattr(shape, "placeholder_format")
+                            and "title" in str(shape.placeholder_format.type).lower()
+                        ):
+                            slide_data["title"] = text_content
+                        else:
+                            slide_data["content"].append(text_content)
+
+                        total_text_length += len(text_content)
+
+                    # Extract text from tables if present
+                    if hasattr(shape, "table"):
+                        table_text = []
+                        for row in shape.table.rows:
+                            row_text = []
+                            for cell in row.cells:
+                                if cell.text.strip():
+                                    row_text.append(cell.text.strip())
+                            if row_text:
+                                table_text.append(" | ".join(row_text))
+                        if table_text:
+                            slide_data["content"].append("\n".join(table_text))
+
+                # Extract speaker notes if requested
+                if extract_notes and hasattr(slide, "notes_slide"):
+                    try:
+                        if (
+                            hasattr(slide.notes_slide, "notes_text_frame")
+                            and slide.notes_slide.notes_text_frame.text.strip()
+                        ):
+                            slide_data["notes"] = slide.notes_slide.notes_text_frame.text.strip()
+                            total_text_length += len(slide_data["notes"])
+                    except Exception as e:
+                        self.logger.warning(f"Failed to extract notes from slide {slide_idx + 1}: {e}")
+
+                slides_content.append(slide_data)
+
+            # Extract presentation structure
+            structure = self._extract_slide_structure(presentation)
+
+            processing_time = time.time() - start_time
+
+            return {
+                "slides": slides_content,
+                "structure": structure,
+                "processing_time": processing_time,
+                "total_text_length": total_text_length,
+                "slide_count": len(slides_content),
+            }
+
+        except Exception as e:
+            self.logger.error(f"Failed to extract content from PPTX: {e}")
+            raise
+
+    def _format_content_for_llm(
+        self, extraction_result: dict[str, Any], output_format: str, include_structure: bool = True
+    ) -> str:
+        """Format extracted PPTX content to be LLM-friendly.
+
+        Args:
+            extraction_result: Dictionary containing extracted content
+            output_format: Desired output format
+            include_structure: Whether to include presentation structure information
+
+        Returns:
+            Formatted content string
+        """
+        slides = extraction_result["slides"]
+        structure = extraction_result["structure"]
+
+        if output_format.lower() == "markdown":
+            content_parts = []
+
+            if include_structure:
+                content_parts.append("# Presentation Overview")
+                content_parts.append(f"- **Total Slides**: {structure['slide_count']}")
+                if structure.get("slide_sizes"):
+                    sizes = structure["slide_sizes"]
+                    content_parts.append(
+                        f'- **Slide Size**: {sizes["width_inches"]:.1f}" × {sizes["height_inches"]:.1f}"'
+                    )
+                content_parts.append(f"- **Has Speaker Notes**: {'Yes' if structure['has_notes'] else 'No'}")
+                content_parts.append("")
+
+            # Format each slide
+            for slide in slides:
+                content_parts.append(f"## Slide {slide['slide_number']}")
+
+                if slide["title"]:
+                    content_parts.append(f"**Title**: {slide['title']}")
+                    content_parts.append("")
+
+                if slide["content"]:
+                    content_parts.append("**Content**:")
+                    for content_item in slide["content"]:
+                        # Format multi-line content properly
+                        for line in content_item.split("\n"):
+                            if line.strip():
+                                content_parts.append(f"- {line.strip()}")
+                    content_parts.append("")
+
+                if slide["notes"]:
+                    content_parts.append("**Speaker Notes**:")
+                    content_parts.append(slide["notes"])
+                    content_parts.append("")
+
+                content_parts.append("---")
+                content_parts.append("")
+
+            return "\n".join(content_parts)
+
+        elif output_format.lower() == "json":
+            return json.dumps(
+                {
+                    "presentation_structure": structure if include_structure else None,
+                    "slides": slides,
+                    "metadata": {
+                        "total_slides": len(slides),
+                        "total_text_length": extraction_result["total_text_length"],
+                        "processing_time": extraction_result["processing_time"],
+                    },
+                },
+                indent=2,
+            )
+
+        elif output_format.lower() == "html":
+            html_parts = ["<div class='presentation'>"]
+
+            if include_structure:
+                html_parts.append("<div class='presentation-overview'>")
+                html_parts.append("<h1>Presentation Overview</h1>")
+                html_parts.append(f"<p><strong>Total Slides:</strong> {structure['slide_count']}</p>")
+                if structure.get("slide_sizes"):
+                    sizes = structure["slide_sizes"]
+                    html_parts.append(
+                        "<p>"
+                        "<strong>Slide Size:</strong> "
+                        f"{sizes['width_inches']:.1f} × {sizes['height_inches']:.1f}"
+                        "</p>"
+                    )
+                html_parts.append(
+                    f"<p><strong>Has Speaker Notes:</strong> {'Yes' if structure['has_notes'] else 'No'}</p>"
+                )
+                html_parts.append("</div>")
+
+            for slide in slides:
+                html_parts.append(f"<div class='slide' data-slide='{slide['slide_number']}'>")
+                html_parts.append(f"<h2>Slide {slide['slide_number']}</h2>")
+
+                if slide["title"]:
+                    html_parts.append(f"<h3>{slide['title']}</h3>")
+
+                if slide["content"]:
+                    html_parts.append("<div class='slide-content'>")
+                    for content_item in slide["content"]:
+                        html_parts.append(f"<p>{content_item.replace(chr(10), '<br>')}</p>")
+                    html_parts.append("</div>")
+
+                if slide["notes"]:
+                    html_parts.append(
+                        "<div class='speaker-notes'>"
+                        "<strong>Speaker Notes:</strong>"
+                        f"<br>{slide['notes'].replace(chr(10), '<br>')}"
+                        "</div>"
+                    )
+
+                html_parts.append("</div>")
+
+            html_parts.append("</div>")
+            return "\n".join(html_parts)
+
+        else:  # Plain text
+            text_parts = []
+
+            if include_structure:
+                text_parts.append("PRESENTATION OVERVIEW")
+                text_parts.append(f"Total Slides: {structure['slide_count']}")
+                if structure.get("slide_sizes"):
+                    sizes = structure["slide_sizes"]
+                    text_parts.append(f'Slide Size: {sizes["width_inches"]:.1f}" × {sizes["height_inches"]:.1f}"')
+                text_parts.append(f"Has Speaker Notes: {'Yes' if structure['has_notes'] else 'No'}")
+                text_parts.append("\n" + "=" * 50 + "\n")
+
+            for slide in slides:
+                text_parts.append(f"SLIDE {slide['slide_number']}")
+
+                if slide["title"]:
+                    text_parts.append(f"Title: {slide['title']}")
+
+                if slide["content"]:
+                    text_parts.append("Content:")
+                    for content_item in slide["content"]:
+                        text_parts.append(f"  {content_item}")
+
+                if slide["notes"]:
+                    text_parts.append(f"Speaker Notes: {slide['notes']}")
+
+                text_parts.append("\n" + "-" * 30 + "\n")
+
+            return "\n".join(text_parts)
+
+    def mcp_extract_pptx_content(
+        self,
+        file_path: str = Field(description="Path to the PPTX/PPT presentation file to extract content from"),
+        output_format: Literal["markdown", "json", "html", "text"] = Field(
+            default="markdown", description="Output format: 'markdown', 'json', 'html', or 'text'"
+        ),
+        extract_images: bool = Field(
+            default=True, description="Whether to extract and save images from the presentation"
+        ),
+        extract_notes: bool = Field(default=True, description="Whether to extract speaker notes"),
+        include_structure: bool = Field(
+            default=True, description="Whether to include presentation structure information"
+        ),
+    ) -> ActionResponse:
+        """Extract content from PPTX/PPT presentations using python-pptx.
+
+        This tool provides comprehensive PowerPoint presentation content extraction with support for:
+        - PPTX and PPT files
+        - Text extraction from slides, titles, and content
+        - Speaker notes extraction
+        - Image and media extraction
+        - Presentation structure analysis
+        - Multiple output formats (Markdown, JSON, HTML, Text)
+        - LLM-optimized formatting
+
+        Args:
+            file_path: Path to the PPTX/PPT presentation file
+            output_format: Desired output format
+            extract_images: Extract embedded media files
+            extract_notes: Extract speaker notes
+            include_structure: Include presentation structure info
+
+        Returns:
+            ActionResponse with extracted content, metadata, and media file paths
+        """
+        try:
+            # Handle FieldInfo objects from pydantic
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+            if isinstance(output_format, FieldInfo):
+                output_format = output_format.default
+            if isinstance(extract_images, FieldInfo):
+                extract_images = extract_images.default
+            if isinstance(extract_notes, FieldInfo):
+                extract_notes = extract_notes.default
+            if isinstance(include_structure, FieldInfo):
+                include_structure = include_structure.default
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Processing PPTX presentation: {file_path.name}", Color.cyan)
+
+            # Extract embedded media if requested
+            saved_media = []
+            if extract_images and file_path.suffix.lower() == ".pptx":
+                saved_media = self._extract_images_from_pptx(file_path, file_path.stem)
+
+            # Extract presentation content
+            extraction_result = self._extract_content_from_pptx(file_path, extract_notes=extract_notes)
+
+            # Format content for LLM consumption
+            formatted_content = self._format_content_for_llm(
+                extraction_result, output_format, include_structure=include_structure
+            )
+
+            # Prepare metadata
+            file_stats = file_path.stat()
+            document_metadata = DocumentMetadata(
+                file_name=file_path.name,
+                file_size=file_stats.st_size,
+                file_type=file_path.suffix.lower(),
+                absolute_path=str(file_path.absolute()),
+                page_count=extraction_result["slide_count"],  # Use slide count as page count
+                processing_time=extraction_result["processing_time"],
+                extracted_images=[media["path"] for media in saved_media if media["type"] == "image"],
+                extracted_media=saved_media,
+                output_format=output_format,
+                llm_enhanced=False,
+                ocr_applied=False,
+            )
+
+            # Add PPTX-specific metadata
+            pptx_metadata = {
+                "slide_count": extraction_result["slide_count"],
+                "total_text_length": extraction_result["total_text_length"],
+                "has_speaker_notes": extraction_result["structure"]["has_notes"],
+                "slide_layouts": extraction_result["structure"]["slide_layouts"],
+                "presentation_size": extraction_result["structure"].get("slide_sizes"),
+                "media_files_count": len(saved_media),
+            }
+
+            # Merge metadata
+            final_metadata = {**document_metadata.model_dump(), **pptx_metadata}
+
+            self._color_log(
+                f"Successfully extracted content from {file_path.name} "
+                f"({extraction_result['slide_count']} slides, "
+                f"{len(formatted_content)} characters, {len(saved_media)} media files)",
+                Color.green,
+            )
+
+            return ActionResponse(success=True, message=formatted_content, metadata=final_metadata)
+
+        except FileNotFoundError as e:
+            self.logger.error(f"File not found: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False, message=f"File not found: {str(e)}", metadata={"error_type": "file_not_found"}
+            )
+        except ValueError as e:
+            self.logger.error(f"Invalid input: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False, message=f"Invalid input: {str(e)}", metadata={"error_type": "invalid_input"}
+            )
+        except Exception as e:
+            self.logger.error(f"PPTX extraction failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"PPTX extraction failed: {str(e)}",
+                metadata={"error_type": "extraction_error"},
+            )
+
+    def mcp_list_supported_formats(self) -> ActionResponse:
+        """List all supported presentation formats for extraction.
+
+        Returns:
+            ActionResponse with list of supported file formats and their descriptions
+        """
+        supported_formats = {
+            "PPTX": "Microsoft PowerPoint Presentation (.pptx) - full support",
+            "PPT": "Microsoft PowerPoint Presentation (.ppt) - limited support",
+        }
+
+        format_list = "\n".join(
+            [f"**{format_name}**: {description}" for format_name, description in supported_formats.items()]
+        )
+
+        return ActionResponse(
+            success=True,
+            message=f"Supported presentation formats:\n\n{format_list}",
+            metadata={"supported_formats": list(supported_formats.keys()), "total_formats": len(supported_formats)},
+        )
+
+
+# Example usage and entry point
+if __name__ == "__main__":
+    load_dotenv()
+
+    # Default arguments for testing
+    args = ActionArguments(
+        name="pptx_extraction_service",
+        transport="stdio",
+        workspace=os.getenv("AWORLD_WORKSPACE", "~"),
+    )
+
+    # Initialize and run the PPTX extraction service
+    try:
+        service = PPTXExtractionCollection(args)
+        service.run()
+    except Exception as e:
+        print(f"An error occurred: {e}: {traceback.format_exc()}")

chapter9/gaia-experience/AWorld/examples/gaia/mcp_collections/documents/msxlsx.py

@@ -0,0 +1,680 @@
+import json
+import os
+import subprocess
+import sys
+import time
+import traceback
+import zipfile
+from pathlib import Path
+from typing import Any, Literal
+
+import pandas as pd
+from dotenv import load_dotenv
+from openpyxl import load_workbook
+from pydantic import Field
+from pydantic.fields import FieldInfo
+
+from aworld.logs.util import Color
+from examples.gaia.mcp_collections.base import ActionArguments, ActionCollection, ActionResponse
+from examples.gaia.mcp_collections.documents.models import DocumentMetadata
+
+
+class XLSXExtractionCollection(ActionCollection):
+    """MCP service for Excel document content extraction using xlrd and pandas.
+
+    Supports extraction from XLSX and XLS files.
+    Provides LLM-friendly text output with structured metadata and media file handling.
+    Extracts worksheets, formulas, charts, and embedded images.
+    Includes screenshot functionality for visual representation of Excel data.
+    """
+
+    def __init__(self, arguments: ActionArguments) -> None:
+        super().__init__(arguments)
+        self._media_output_dir = self.workspace / "extracted_media"
+        self._media_output_dir.mkdir(exist_ok=True)
+
+        # Create screenshots directory
+        self._screenshots_dir = self.workspace / "excel_screenshots"
+        self._screenshots_dir.mkdir(exist_ok=True)
+
+        self.supported_extensions: set = {
+            ".xlsx",
+            ".xls",
+        }
+
+        self._color_log("Excel Extraction Service initialized", Color.green, "debug")
+        self._color_log(f"Media output directory: {self._media_output_dir}", Color.blue, "debug")
+        self._color_log(f"Screenshots directory: {self._screenshots_dir}", Color.blue, "debug")
+
+    def _create_excel_screenshot(self, file_path: Path, sheet_name: str = None) -> str:
+        """Create a JPEG screenshot of the valid Excel area using pyautogui.
+
+        Args:
+            file_path: Path to the Excel file
+            sheet_name: Specific sheet to screenshot (None for first sheet)
+
+        Returns:
+            Path to the generated JPEG screenshot
+        """
+        try:
+            import pyautogui
+
+            # Generate unique filename
+            timestamp = int(time.time())
+            screenshot_filename = f"{file_path.stem}_{sheet_name or 'sheet'}_{timestamp}.jpg"
+            screenshot_path = self._screenshots_dir / screenshot_filename
+
+            # Open Excel file with default application
+            if sys.platform == "darwin":  # macOS
+                subprocess.run(["open", str(file_path)], check=True)
+            elif sys.platform == "win32":  # Windows
+                subprocess.run(["start", str(file_path)], shell=True, check=True)
+            else:  # Linux
+                subprocess.run(["xdg-open", str(file_path)], check=True)
+
+            # Wait for Excel to open
+            time.sleep(3)
+
+            # Take screenshot of the entire screen
+            screenshot = pyautogui.screenshot()
+
+            # Convert RGBA to RGB before saving as JPEG
+            if screenshot.mode == "RGBA":
+                screenshot = screenshot.convert("RGB")
+
+            screenshot.save(screenshot_path, "JPEG", quality=95)
+
+            self._color_log(f"Created Excel screenshot: {screenshot_filename}", Color.green)
+            return str(screenshot_path)
+
+        except Exception as e:
+            self.logger.error(f"Failed to create Excel screenshot with pyautogui: {str(e)}")
+            raise
+
+    def _extract_embedded_media_xlsx(self, file_path: Path) -> list[dict[str, str]]:
+        """Extract embedded media from XLSX files.
+
+        Args:
+            file_path: Path to the XLSX file
+
+        Returns:
+            List of dictionaries containing media information
+        """
+        saved_media = []
+
+        try:
+            # Load workbook to extract images
+            workbook = load_workbook(file_path, data_only=False)
+
+            for sheet_name in workbook.sheetnames:
+                worksheet = workbook[sheet_name]
+
+                # Extract images from worksheet
+                if hasattr(worksheet, "_images"):
+                    for idx, image in enumerate(worksheet._images):
+                        try:
+                            # Generate unique filename
+                            image_filename = f"{file_path.stem}_{sheet_name}_img_{idx}.png"
+                            image_path = self._media_output_dir / image_filename
+
+                            # Save image
+                            if hasattr(image, "ref"):
+                                # Extract image data
+                                img_data = image._data()
+                                if img_data:
+                                    with open(image_path, "wb") as f:
+                                        f.write(img_data)
+
+                                    saved_media.append(
+                                        {
+                                            "type": "image",
+                                            "path": str(image_path),
+                                            "sheet": sheet_name,
+                                            "filename": image_filename,
+                                        }
+                                    )
+
+                                    self._color_log(f"Saved image: {image_filename}", Color.blue)
+                        except Exception as e:
+                            self.logger.warning(f"Failed to extract image {idx} from sheet {sheet_name}: {str(e)}")
+
+            # Also try to extract from ZIP structure for additional media
+            with zipfile.ZipFile(file_path, "r") as zip_file:
+                media_files = [f for f in zip_file.namelist() if f.startswith("xl/media/")]
+
+                for media_file in media_files:
+                    try:
+                        media_data = zip_file.read(media_file)
+                        media_filename = f"{file_path.stem}_{Path(media_file).name}"
+                        media_path = self._media_output_dir / media_filename
+
+                        with open(media_path, "wb") as f:
+                            f.write(media_data)
+
+                        # Determine media type based on extension
+                        media_ext = Path(media_file).suffix.lower()
+                        if media_ext in [".png", ".jpg", ".jpeg", ".gif", ".bmp"]:
+                            media_type = "image"
+                        elif media_ext in [".mp3", ".wav", ".m4a"]:
+                            media_type = "audio"
+                        elif media_ext in [".mp4", ".avi", ".mov"]:
+                            media_type = "video"
+                        else:
+                            media_type = "other"
+
+                        saved_media.append(
+                            {
+                                "type": media_type,
+                                "path": str(media_path),
+                                "filename": media_filename,
+                                "original_path": media_file,
+                            }
+                        )
+
+                        self._color_log(f"Saved media: {media_filename}", Color.blue)
+                    except Exception as e:
+                        self.logger.warning(f"Failed to extract media {media_file}: {str(e)}")
+
+        except Exception as e:
+            self.logger.warning(f"Failed to extract media from XLSX: {str(e)}")
+
+        return saved_media
+
+    def _extract_excel_content(self, file_path: Path, sheet_names: list[str] | None = None) -> dict[str, Any]:
+        """Extract content from Excel files using pandas and xlrd.
+
+        Args:
+            file_path: Path to the Excel file
+            sheet_names: Specific sheets to process (None for all sheets)
+
+        Returns:
+            Dictionary containing extracted content and metadata
+        """
+        start_time = time.time()
+
+        try:
+            # Determine file type and read accordingly
+            if file_path.suffix.lower() == ".xlsx":
+                # Use openpyxl engine for XLSX files
+                excel_file = pd.ExcelFile(file_path, engine="openpyxl")
+            else:
+                # Use xlrd engine for XLS files
+                excel_file = pd.ExcelFile(file_path, engine="xlrd")
+
+            # Get all sheet names if not specified
+            if sheet_names is None:
+                sheet_names = excel_file.sheet_names
+
+            sheets_data = {}
+            total_rows = 0
+            total_cols = 0
+
+            # Extract data from each sheet
+            for sheet_name in sheet_names:
+                if sheet_name in excel_file.sheet_names:
+                    try:
+                        # Read sheet data
+                        df = pd.read_excel(excel_file, sheet_name=sheet_name, header=None)
+
+                        # Remove completely empty rows and columns
+                        df = df.dropna(how="all").dropna(axis=1, how="all")
+
+                        if not df.empty:
+                            sheets_data[sheet_name] = {
+                                "data": df,
+                                "shape": df.shape,
+                                "columns": df.columns.tolist(),
+                                "non_empty_cells": df.count().sum(),
+                            }
+
+                            total_rows += df.shape[0]
+                            total_cols = max(total_cols, df.shape[1])
+                        else:
+                            sheets_data[sheet_name] = {"data": df, "shape": (0, 0), "columns": [], "non_empty_cells": 0}
+
+                    except Exception as e:
+                        self.logger.warning(f"Failed to read sheet '{sheet_name}': {str(e)}")
+                        sheets_data[sheet_name] = {
+                            "error": str(e),
+                            "shape": (0, 0),
+                            "columns": [],
+                            "non_empty_cells": 0,
+                        }
+
+            processing_time = time.time() - start_time
+
+            return {
+                "sheets_data": sheets_data,
+                "sheet_names": list(sheets_data.keys()),
+                "total_sheets": len(sheets_data),
+                "total_rows": total_rows,
+                "total_columns": total_cols,
+                "processing_time": processing_time,
+                "file_engine": "openpyxl" if file_path.suffix.lower() == ".xlsx" else "xlrd",
+            }
+
+        except Exception as e:
+            self.logger.error(f"Failed to extract Excel content: {str(e)}")
+            raise
+
+    def _format_content_for_llm(
+        self, extraction_result: dict[str, Any], output_format: str, include_empty_cells: bool = False
+    ) -> str:
+        """Format extracted Excel content to be LLM-friendly.
+
+        Args:
+            extraction_result: Result from _extract_excel_content
+            output_format: Desired output format
+            include_empty_cells: Whether to include empty cells in output
+
+        Returns:
+            Formatted content string
+        """
+        sheets_data = extraction_result["sheets_data"]
+
+        if output_format.lower() == "markdown":
+            content_parts = []
+            content_parts.append("# Excel Document Content\n")
+            content_parts.append(f"**Total Sheets:** {extraction_result['total_sheets']}\n")
+            content_parts.append(f"**Processing Engine:** {extraction_result['file_engine']}\n\n")
+
+            for sheet_name, sheet_info in sheets_data.items():
+                content_parts.append(f"## Sheet: {sheet_name}\n")
+
+                if "error" in sheet_info:
+                    content_parts.append(f"**Error:** {sheet_info['error']}\n\n")
+                    continue
+
+                df: pd.DataFrame = sheet_info["data"]
+                shape = sheet_info["shape"]
+
+                content_parts.append(f"**Dimensions:** {shape[0]} rows × {shape[1]} columns\n")
+                content_parts.append(f"**Non-empty cells:** {sheet_info['non_empty_cells']}\n\n")
+
+                if not df.empty:
+                    # Convert DataFrame to markdown table
+                    if include_empty_cells:
+                        # Fill NaN values with empty string for display
+                        df_display = df.fillna("")
+                    else:
+                        # Keep NaN values as they are
+                        df_display = df
+
+                    # Convert to markdown table
+                    try:
+                        markdown_table = df_display.to_markdown(index=False, tablefmt="pipe")
+                        content_parts.append(f"### Data:\n{markdown_table}\n\n")
+                    except Exception:
+                        # Fallback to string representation
+                        content_parts.append(f"### Data (text format):\n```\n{df_display.to_string()}\n```\n\n")
+                else:
+                    content_parts.append("*Sheet is empty*\n\n")
+
+            return "".join(content_parts)
+
+        elif output_format.lower() == "json":
+            json_data = {
+                "document_info": {
+                    "total_sheets": extraction_result["total_sheets"],
+                    "total_rows": extraction_result["total_rows"],
+                    "total_columns": extraction_result["total_columns"],
+                    "processing_engine": extraction_result["file_engine"],
+                },
+                "sheets": {},
+            }
+
+            for sheet_name, sheet_info in sheets_data.items():
+                if "error" in sheet_info:
+                    json_data["sheets"][sheet_name] = {"error": sheet_info["error"], "shape": sheet_info["shape"]}
+                    continue
+
+                df = sheet_info["data"]
+
+                if not df.empty:
+                    # Convert DataFrame to records
+                    if include_empty_cells:
+                        df_records = df.fillna("").to_dict("records")
+                    else:
+                        df_records = df.to_dict("records")
+
+                    json_data["sheets"][sheet_name] = {
+                        "shape": sheet_info["shape"],
+                        "non_empty_cells": sheet_info["non_empty_cells"],
+                        "data": df_records,
+                    }
+                else:
+                    json_data["sheets"][sheet_name] = {"shape": sheet_info["shape"], "non_empty_cells": 0, "data": []}
+
+            return json.dumps(json_data, indent=2, default=str)
+
+        elif output_format.lower() == "html":
+            html_parts = []
+            html_parts.append("<html><body>")
+            html_parts.append("<h1>Excel Document Content</h1>")
+            html_parts.append(f"<p><strong>Total Sheets:</strong> {extraction_result['total_sheets']}</p>")
+            html_parts.append(f"<p><strong>Processing Engine:</strong> {extraction_result['file_engine']}</p>")
+
+            for sheet_name, sheet_info in sheets_data.items():
+                html_parts.append(f"<h2>Sheet: {sheet_name}</h2>")
+
+                if "error" in sheet_info:
+                    html_parts.append(f"<p><strong>Error:</strong> {sheet_info['error']}</p>")
+                    continue
+
+                df = sheet_info["data"]
+                shape = sheet_info["shape"]
+
+                html_parts.append(f"<p><strong>Dimensions:</strong> {shape[0]} rows × {shape[1]} columns</p>")
+                html_parts.append(f"<p><strong>Non-empty cells:</strong> {sheet_info['non_empty_cells']}</p>")
+
+                if not df.empty:
+                    # Convert DataFrame to HTML table
+                    if include_empty_cells:
+                        df_display = df.fillna("")
+                    else:
+                        df_display = df
+
+                    html_table = df_display.to_html(index=False, escape=False, table_id=f"sheet_{sheet_name}")
+                    html_parts.append(html_table)
+                else:
+                    html_parts.append("<p><em>Sheet is empty</em></p>")
+
+            html_parts.append("</body></html>")
+            return "".join(html_parts)
+
+        else:  # text format
+            content_parts = []
+            content_parts.append(f"Excel Document Content\n{'=' * 50}\n")
+            content_parts.append(f"Total Sheets: {extraction_result['total_sheets']}\n")
+            content_parts.append(f"Processing Engine: {extraction_result['file_engine']}\n\n")
+
+            for sheet_name, sheet_info in sheets_data.items():
+                content_parts.append(f"Sheet: {sheet_name}\n{'-' * 30}\n")
+
+                if "error" in sheet_info:
+                    content_parts.append(f"Error: {sheet_info['error']}\n\n")
+                    continue
+
+                df = sheet_info["data"]
+                shape = sheet_info["shape"]
+
+                content_parts.append(f"Dimensions: {shape[0]} rows × {shape[1]} columns\n")
+                content_parts.append(f"Non-empty cells: {sheet_info['non_empty_cells']}\n\n")
+
+                if not df.empty:
+                    if include_empty_cells:
+                        df_display = df.fillna("")
+                    else:
+                        df_display = df
+
+                    content_parts.append(f"Data:\n{df_display.to_string()}\n\n")
+                else:
+                    content_parts.append("Sheet is empty\n\n")
+
+            return "".join(content_parts)
+
+    def mcp_extract_excel_content(
+        self,
+        file_path: str = Field(description="Path to the Excel document file to extract content from"),
+        output_format: Literal["markdown", "json", "html", "text"] = Field(
+            default="markdown", description="Output format: 'markdown', 'json', 'html', or 'text'"
+        ),
+        extract_images: bool = Field(default=True, description="Whether to extract and save images from the document"),
+        create_screenshot: bool = Field(
+            default=False, description="Whether to create a JPEG screenshot of the Excel data"
+        ),
+        sheet_names: str | None = Field(
+            default=None, description="Comma-separated list of specific sheet names to process (None for all sheets)"
+        ),
+        include_empty_cells: bool = Field(default=False, description="Whether to include empty cells in the output"),
+        screenshot_max_rows: int = Field(default=50, description="Maximum rows to include in screenshot"),
+        screenshot_max_cols: int = Field(default=20, description="Maximum columns to include in screenshot"),
+    ) -> ActionResponse:
+        """Extract content from Excel documents using pandas and xlrd.
+
+        This tool provides comprehensive Excel document content extraction with support for:
+        - XLSX and XLS files
+        - Multiple worksheets
+        - Text and numeric data extraction
+        - Image and media extraction (XLSX only)
+        - JPEG screenshot generation of Excel data
+        - Metadata collection
+        - LLM-optimized output formatting
+
+        Args:
+            file_path: Path to the Excel file
+            output_format: Desired output format
+            extract_images: Whether to extract embedded images
+            create_screenshot: Whether to create a JPEG screenshot
+            sheet_names: Specific sheets to process
+            include_empty_cells: Whether to include empty cells
+            screenshot_max_rows: Maximum rows in screenshot
+            screenshot_max_cols: Maximum columns in screenshot
+
+        Returns:
+            ActionResponse with extracted content, metadata, media file paths, and screenshot path
+        """
+        try:
+            # Handle FieldInfo objects
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+            if isinstance(output_format, FieldInfo):
+                output_format = output_format.default
+            if isinstance(extract_images, FieldInfo):
+                extract_images = extract_images.default
+            if isinstance(create_screenshot, FieldInfo):
+                create_screenshot = create_screenshot.default
+            if isinstance(sheet_names, FieldInfo):
+                sheet_names = sheet_names.default
+            if isinstance(include_empty_cells, FieldInfo):
+                include_empty_cells = include_empty_cells.default
+            if isinstance(screenshot_max_rows, FieldInfo):
+                screenshot_max_rows = screenshot_max_rows.default
+            if isinstance(screenshot_max_cols, FieldInfo):
+                screenshot_max_cols = screenshot_max_cols.default
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Processing Excel document: {file_path.name}", Color.cyan)
+
+            # Parse sheet names if provided
+            target_sheets = None
+            if sheet_names:
+                target_sheets = [name.strip() for name in sheet_names.split(",")]
+
+            # Extract content from Excel file
+            extraction_result = self._extract_excel_content(file_path, target_sheets)
+
+            # Extract embedded media if requested (XLSX only)
+            saved_media = []
+            if extract_images and file_path.suffix.lower() == ".xlsx":
+                saved_media = self._extract_embedded_media_xlsx(file_path)
+            elif extract_images and file_path.suffix.lower() == ".xls":
+                self._color_log("Image extraction not supported for XLS files", Color.yellow)
+
+            # Create screenshot if requested
+            screenshot_path = None
+            if create_screenshot:
+                target_sheet = target_sheets[0] if target_sheets else None
+                screenshot_path = self._create_excel_screenshot(file_path, target_sheet)
+
+            # Format content for LLM consumption
+            formatted_content = self._format_content_for_llm(extraction_result, output_format, include_empty_cells)
+
+            # Prepare metadata
+            file_stats = file_path.stat()
+
+            # Create Excel-specific metadata
+            excel_metadata = {
+                "sheet_count": extraction_result["total_sheets"],
+                "sheet_names": extraction_result["sheet_names"],
+                "total_rows": extraction_result["total_rows"],
+                "total_columns": extraction_result["total_columns"],
+                "processing_engine": extraction_result["file_engine"],
+                "extracted_images": [media["path"] for media in saved_media if media["type"] == "image"],
+                "extracted_media": saved_media,
+                "screenshot_path": screenshot_path,
+                "include_empty_cells": include_empty_cells,
+                "processed_sheets": target_sheets or extraction_result["sheet_names"],
+            }
+
+            document_metadata = DocumentMetadata(
+                file_name=file_path.name,
+                file_size=file_stats.st_size,
+                file_type=file_path.suffix.lower(),
+                absolute_path=str(file_path.absolute()),
+                page_count=extraction_result["total_sheets"],  # Use sheet count as "page" count
+                processing_time=extraction_result["processing_time"],
+                extracted_images=[media["path"] for media in saved_media if media["type"] == "image"],
+                extracted_media=saved_media,
+                output_format=output_format,
+                llm_enhanced=False,
+                ocr_applied=False,
+            )
+
+            # Combine standard and Excel-specific metadata
+            combined_metadata = document_metadata.model_dump()
+            combined_metadata.update(excel_metadata)
+
+            success_message = (
+                f"Successfully extracted content from {file_path.name} "
+                f"({len(formatted_content)} characters, {extraction_result['total_sheets']} sheets, "
+                f"{len(saved_media)} media files"
+            )
+
+            if screenshot_path:
+                success_message += f", screenshot saved to: {screenshot_path}"
+
+            self._color_log(success_message, Color.green)
+
+            return ActionResponse(success=True, message=formatted_content, metadata=combined_metadata)
+
+        except FileNotFoundError as e:
+            self.logger.error(f"File not found: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False, message=f"File not found: {str(e)}", metadata={"error_type": "file_not_found"}
+            )
+        except ValueError as e:
+            self.logger.error(f"Invalid input: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"Invalid input: {str(e)}",
+                metadata={"error_type": "invalid_input"},
+            )
+        except Exception as e:
+            self.logger.error(f"Excel extraction failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"Excel extraction failed: {str(e)}",
+                metadata={"error_type": "extraction_error"},
+            )
+
+    def mcp_create_excel_screenshot(
+        self,
+        file_path: str = Field(description="Path to the Excel document file"),
+        sheet_name: str | None = Field(
+            default=None, description="Specific sheet name to screenshot (None for first sheet)"
+        ),
+        max_rows: int = Field(default=50, description="Maximum number of rows to include"),
+        max_cols: int = Field(default=20, description="Maximum number of columns to include"),
+    ) -> ActionResponse:
+        """Create a JPEG screenshot of the valid Excel area.
+
+        This tool creates a visual representation of Excel data as a JPEG image,
+        useful for further image processing or visual analysis.
+
+        Args:
+            file_path: Path to the Excel file
+            sheet_name: Specific sheet to screenshot
+            max_rows: Maximum rows to include in screenshot
+            max_cols: Maximum columns to include in screenshot
+
+        Returns:
+            ActionResponse with screenshot file path and metadata
+        """
+        try:
+            # Handle FieldInfo objects
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+            if isinstance(sheet_name, FieldInfo):
+                sheet_name = sheet_name.default
+            if isinstance(max_rows, FieldInfo):
+                max_rows = max_rows.default
+            if isinstance(max_cols, FieldInfo):
+                max_cols = max_cols.default
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Creating screenshot for Excel document: {file_path.name}", Color.cyan)
+
+            # Create screenshot
+            screenshot_path = self._create_excel_screenshot(file_path, sheet_name)
+
+            # Prepare metadata
+            file_stats = file_path.stat()
+            screenshot_stats = Path(screenshot_path).stat()
+
+            metadata = {
+                "source_file": str(file_path.absolute()),
+                "source_file_size": file_stats.st_size,
+                "screenshot_path": screenshot_path,
+                "screenshot_size": screenshot_stats.st_size,
+                "sheet_name": sheet_name,
+                "max_rows_displayed": max_rows,
+                "max_cols_displayed": max_cols,
+                "format": "JPEG",
+            }
+
+            return ActionResponse(
+                success=True,
+                message=f"Excel screenshot created successfully. File saved to: {screenshot_path}",
+                metadata=metadata,
+            )
+
+        except Exception as e:
+            self.logger.error(f"Screenshot creation failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"Screenshot creation failed: {str(e)}",
+                metadata={"error_type": "screenshot_error"},
+            )
+
+    def mcp_list_supported_formats(self) -> ActionResponse:
+        """List all supported Excel formats for extraction.
+
+        Returns:
+            ActionResponse with list of supported file formats and their descriptions
+        """
+        supported_formats = {
+            "XLSX": "Excel 2007+ format files (.xlsx) - Full support including images",
+            "XLS": "Excel 97-2003 format files (.xls) - Text and data only",
+        }
+
+        format_list = "\n".join(
+            [f"**{format_name}**: {description}" for format_name, description in supported_formats.items()]
+        )
+
+        return ActionResponse(
+            success=True,
+            message=f"Supported Excel formats:\n\n{format_list}",
+            metadata={"supported_formats": list(supported_formats.keys()), "total_formats": len(supported_formats)},
+        )
+
+
+# Example usage and entry point
+if __name__ == "__main__":
+    load_dotenv()
+
+    # Default arguments for testing
+    args = ActionArguments(
+        name="excel_extraction_service",
+        transport="stdio",
+        workspace=os.getenv("AWORLD_WORKSPACE", "/tmp"),
+    )
+
+    # Initialize and run the Excel extraction service
+    try:
+        service = XLSXExtractionCollection(args)
+        service.run()
+    except Exception as e:
+        print(f"An error occurred: {e}: {traceback.format_exc()}")

chapter9/gaia-experience/AWorld/examples/gaia/mcp_collections/documents/pdf.py

@@ -0,0 +1,341 @@
+import json
+import os
+import time
+import traceback
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Literal, Optional  # Added Optional
+
+import markdown
+from dotenv import load_dotenv
+from marker.converters.pdf import PdfConverter
+from marker.models import create_model_dict
+from marker.output import text_from_rendered
+from marker.settings import settings
+from pydantic import Field
+from pydantic.fields import FieldInfo
+
+from aworld.logs.util import Color
+from examples.gaia.mcp_collections.base import ActionArguments, ActionCollection, ActionResponse
+from examples.gaia.mcp_collections.documents.models import DocumentMetadata
+
+
+class DocumentExtractionCollection(ActionCollection):
+    """MCP service for PDF document content extraction using marker package.
+
+    Supports extraction from PDF files only.
+    Provides LLM-friendly text output with structured metadata and media file handling.
+    """
+
+    def __init__(self, arguments: ActionArguments) -> None:
+        super().__init__(arguments)
+        self._models_loaded = False
+        self._marker_models = None
+        self._media_output_dir = self.workspace / "extracted_media"
+        self._media_output_dir.mkdir(exist_ok=True)
+        self._extracted_texts_dir = self.workspace / "extracted_texts"  # New directory for text files
+        self._extracted_texts_dir.mkdir(exist_ok=True)
+
+        self.supported_extensions = {".pdf"}
+
+        self._color_log("PDF Extraction Service initialized", Color.green, "debug")
+        self._color_log(f"Media output directory: {self._media_output_dir}", Color.blue, "debug")
+
+    def _load_marker_models(self) -> None:
+        """Load marker models for document processing.
+
+        Lazy loading to avoid unnecessary resource consumption.
+        """
+        if not self._models_loaded:
+            try:
+                self._color_log("Loading marker models...", Color.yellow)
+                self._marker_models = create_model_dict()
+                self._models_loaded = True
+                self._color_log("Marker models loaded successfully", Color.green)
+            except Exception as e:
+                self.logger.error(f"Failed to load marker models: {str(e)}")
+                raise
+
+    def _extract_content_with_marker(
+        self, file_path: Path, page_range: str | None, force_ocr: bool = False
+    ) -> dict[str, Any]:
+        """Extract content using marker package.
+
+        Args:
+            file_path: Path to the document file
+            page_range: Specific pages to process (e.g., '0,5-10,20')
+            force_ocr: Use OCR to extract text from images if available
+
+        Returns:
+            Dictionary containing extracted content and metadata
+        """
+        start_time = time.time()
+
+        # Prepare marker arguments
+        marker_args = {
+            "fname": str(file_path),
+            "model_lst": self._marker_models,
+            "max_pages": None,
+            "langs": None,
+            "batch_multiplier": 1,
+            "force_ocr": force_ocr,
+        }
+
+        # Handle page range
+        if page_range:
+            # Parse page range string (e.g., "0,5-10,20")
+            pages = []
+            for part in page_range.split(","):
+                if "-" in part:
+                    start, end = map(int, part.split("-"))
+                    pages.extend(range(start, end + 1))
+                else:
+                    pages.append(int(part))
+            marker_args["page_range"] = pages
+        converter: PdfConverter = PdfConverter(artifact_dict=self._marker_models)
+        rendered = converter(str(file_path))
+        text, _, images = text_from_rendered(rendered)
+        text = text.encode(settings.OUTPUT_ENCODING, errors="replace").decode(settings.OUTPUT_ENCODING)
+
+        processing_time = time.time() - start_time
+        return {
+            "content": text,
+            "images": images or {},
+            "metadata": defaultdict(),
+            "processing_time": processing_time,
+        }
+
+    def _save_extracted_media(self, images: dict[str, Any], file_stem: str) -> list[dict[str, str]]:
+        """Save extracted images and return their paths.
+
+        Args:
+            images: Dictionary of extracted images from marker
+            file_stem: Base name for saving files
+
+        Returns:
+            list of dictionaries containing media type and file paths
+        """
+        saved_media = []
+
+        for idx, (page_num, image_data) in enumerate(images.items()):
+            try:
+                # Generate unique filename
+                image_filename = f"{file_stem}_page_{page_num}_img_{idx}.png"
+                image_path = self._media_output_dir / image_filename
+
+                # Save image data
+                if hasattr(image_data, "save"):
+                    # PIL Image object
+                    image_data.save(image_path)
+                elif isinstance(image_data, bytes):
+                    # Raw image bytes
+                    with open(image_path, "wb") as f:
+                        f.write(image_data)
+                else:
+                    # Handle other formats
+                    self.logger.warning(f"Unknown image data type for page {page_num}: {type(image_data)}")
+                    continue
+
+                saved_media.append(
+                    {"type": "image", "path": str(image_path), "page": str(page_num), "filename": image_filename}
+                )
+
+                self._color_log(f"Saved image: {image_filename}", Color.blue)
+
+            except Exception as e:
+                self.logger.error(f"Failed to save image from page {page_num}: {str(e)}")
+
+        return saved_media
+
+    def _format_content_for_llm(self, content: str, output_format: str) -> str:
+        """Format extracted content to be LLM-friendly.
+
+        Args:
+            content: Raw extracted content
+            output_format: Desired output format
+
+        Returns:
+            Formatted content string
+        """
+        if output_format.lower() == "markdown":
+            # Content is already in markdown format from marker
+            return content
+        elif output_format.lower() == "json":
+            # Structure content as JSON
+
+            return json.dumps({"content": content, "format": "structured_text"}, indent=2)
+        elif output_format.lower() == "html":
+            # Convert markdown to HTML if needed
+            try:
+                return markdown.markdown(content)
+            except ImportError:
+                self.logger.warning("markdown package not available, returning raw content")
+                return content
+        else:
+            return content
+
+    def mcp_extract_document_content(
+        self,
+        file_path: str = Field(description="Path to the PDF document file to extract content from"),
+        output_format: Literal["markdown", "json", "html"] = Field(
+            default="markdown", description="Output format: 'markdown', 'json', or 'html'"
+        ),
+        extract_images: bool = Field(default=True, description="Whether to extract and save images from the document"),
+        save_extracted_text_to_file: bool = Field(
+            default=False, description="Save extracted text to a local file"
+        ),  # New parameter
+        use_llm: bool = Field(default=False, description="Use LLM for enhanced accuracy (requires additional setup)"),
+        page_range: str | None = Field(default=None, description="Specific pages to process (e.g., '0,5-10,20')"),
+        force_ocr: bool = Field(default=False, description="Force OCR processing on the entire document"),
+        format_lines: bool = Field(
+            default=False, description="Reformat lines using local OCR model for better quality"
+        ),
+    ) -> ActionResponse:
+        """Extract content from PDF documents using marker package.
+
+        This tool provides comprehensive PDF document content extraction with support for:
+        - PDF files
+        - Text extraction with proper formatting
+        - Image and media extraction
+        - Metadata collection
+        - LLM-optimized output formatting
+
+        Args:
+            args: Document extraction arguments including file path and options
+
+        Returns:
+            ActionResponse with extracted content, metadata, and media file paths
+        """
+        try:
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+            if isinstance(output_format, FieldInfo):
+                output_format = output_format.default
+            if isinstance(extract_images, FieldInfo):
+                extract_images = extract_images.default
+            if isinstance(save_extracted_text_to_file, FieldInfo):  # Handle new parameter
+                save_extracted_text_to_file = save_extracted_text_to_file.default
+            if isinstance(page_range, FieldInfo):
+                page_range = page_range.default
+            if isinstance(use_llm, FieldInfo):
+                use_llm = use_llm.default
+            if isinstance(force_ocr, FieldInfo):
+                force_ocr = force_ocr.default
+            if isinstance(format_lines, FieldInfo):
+                format_lines = format_lines.default
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Processing document: {file_path.name}", Color.cyan)
+
+            # Load marker models if needed
+            self._load_marker_models()
+
+            # Extract content using marker
+            extraction_result = self._extract_content_with_marker(file_path, page_range, force_ocr)
+
+            # Save extracted media if requested
+            saved_media = []
+            if extract_images and extraction_result["images"]:
+                saved_media = self._save_extracted_media(extraction_result["images"], file_path.stem)
+
+            # Format content for LLM consumption
+            formatted_content = self._format_content_for_llm(extraction_result["content"], output_format)
+
+            # Save extracted text to file if requested
+            saved_text_path_str: Optional[str] = None
+            if save_extracted_text_to_file:
+                text_file_name = f"{file_path.stem}_extracted_text.txt"
+                saved_text_path = self._extracted_texts_dir / text_file_name
+                try:
+                    with open(saved_text_path, "w", encoding="utf-8") as f:
+                        f.write(formatted_content)
+                    saved_text_path_str = str(saved_text_path.absolute())
+                    self._color_log(f"Saved extracted text to: {saved_text_path_str}", Color.blue)
+                except Exception as e:
+                    self.logger.error(f"Failed to save extracted text to {saved_text_path}: {str(e)}")
+                    # Optionally, you might want to reflect this failure in the response
+
+            # Prepare metadata
+            file_stats = file_path.stat()
+            document_metadata = DocumentMetadata(
+                file_name=file_path.name,
+                file_size=file_stats.st_size,
+                file_type=file_path.suffix.lower(),
+                absolute_path=str(file_path.absolute()),
+                page_count=extraction_result["metadata"].get("page_count"),
+                processing_time=extraction_result["processing_time"],
+                extracted_images=[media["path"] for media in saved_media if media["type"] == "image"],
+                extracted_media=saved_media,
+                output_format=output_format,
+                llm_enhanced=use_llm,
+                ocr_applied=force_ocr or format_lines,
+                extracted_text_file_path=saved_text_path_str,
+            )
+
+            self._color_log(
+                f"Successfully extracted content from {file_path.name} "
+                f"({len(formatted_content)} characters, {len(saved_media)} media files)",
+                Color.green,
+            )
+
+            return ActionResponse(success=True, message=formatted_content, metadata=document_metadata.model_dump())
+
+        except FileNotFoundError as e:
+            self.logger.error(f"File not found: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False, message=f"File not found: {str(e)}", metadata={"error_type": "file_not_found"}
+            )
+        except ValueError as e:
+            self.logger.error(f"Invalid input: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"Invalid input: {str(e)}: {traceback.format_exc()}",
+                metadata={"error_type": "invalid_input"},
+            )
+        except Exception as e:
+            self.logger.error(f"Document extraction failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"Document extraction failed: {str(e)}",
+                metadata={"error_type": "extraction_error"},
+            )
+
+    def mcp_list_supported_formats(self) -> ActionResponse:
+        """list all supported document formats for extraction.
+
+        Returns:
+            ActionResponse with list of supported file formats and their descriptions
+        """
+        supported_formats = {
+            "PDF": "Portable Document Format files (.pdf)",
+        }
+
+        format_list = "\n".join(
+            [f"**{format_name}**: {description}" for format_name, description in supported_formats.items()]
+        )
+
+        return ActionResponse(
+            success=True,
+            message=f"Supported document formats:\n\n{format_list}",
+            metadata={"supported_formats": list(supported_formats.keys()), "total_formats": len(supported_formats)},
+        )
+
+
+# Example usage and entry point
+if __name__ == "__main__":
+    load_dotenv()
+
+    # Default arguments for testing
+    args = ActionArguments(
+        name="document_extraction_service",
+        transport="stdio",
+        workspace=os.getenv("AWORLD_WORKSPACE", "~"),
+    )
+
+    # Initialize and run the document extraction service
+    try:
+        service = DocumentExtractionCollection(args)
+        service.run()
+    except Exception as e:
+        print(f"An error occurred: {e}: {traceback.format_exc()}")

chapter9/gaia-experience/AWorld/examples/gaia/mcp_collections/documents/txt.py

@@ -0,0 +1,648 @@
+import json
+import os
+import time
+import traceback
+from pathlib import Path
+from typing import Any, Literal
+
+import chardet
+from dotenv import load_dotenv
+from pydantic import Field
+from pydantic.fields import FieldInfo
+
+from aworld.logs.util import Color
+from examples.gaia.mcp_collections.base import ActionArguments, ActionCollection, ActionResponse
+from examples.gaia.mcp_collections.documents.models import DocumentMetadata
+from examples.gaia.mcp_collections.utils import get_mime_type
+
+
+class TextExtractionCollection(ActionCollection):
+    """MCP service for text document content extraction.
+
+    Supports extraction from TXT and other raw text format files.
+    Provides LLM-friendly text output with structured metadata and encoding detection.
+    Handles various text encodings and provides comprehensive file analysis.
+    """
+
+    def __init__(self, arguments: ActionArguments) -> None:
+        super().__init__(arguments)
+        self._media_output_dir = self.workspace / "extracted_media"
+        self._media_output_dir.mkdir(exist_ok=True)
+
+        self.supported_extensions: set = {
+            ".txt",
+            ".text",
+            ".log",
+            ".md",
+            ".markdown",
+            ".rst",
+            ".rtf",
+            ".csv",
+            ".tsv",
+            ".json",
+            ".xml",
+            ".yaml",
+            ".yml",
+            ".ini",
+            ".cfg",
+            ".conf",
+            ".properties",
+            ".sql",
+            ".py",
+            ".js",
+            ".html",
+            ".htm",
+            ".css",
+            ".java",
+            ".cpp",
+            ".c",
+            ".h",
+            ".php",
+            ".rb",
+            ".go",
+            ".rs",
+            ".sh",
+            ".bat",
+            ".ps1",
+            ".r",
+            ".m",
+            ".swift",
+            ".kt",
+            ".scala",
+            ".pl",
+            ".lua",
+            ".vim",
+            ".tex",
+            ".bib",
+        }
+
+        self._color_log("Text Extraction Service initialized", Color.green, "debug")
+        self._color_log(f"Media output directory: {self._media_output_dir}", Color.blue, "debug")
+
+    def _validate_file_path(self, file_path: str) -> Path:
+        """Validate and resolve file path.
+
+        Args:
+            file_path: Path to the text document file
+
+        Returns:
+            Resolved Path object
+
+        Raises:
+            FileNotFoundError: If file doesn't exist
+            ValueError: If file type is not supported
+        """
+        path = Path(file_path)
+        if not path.is_absolute():
+            path = self.workspace / path
+
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {path}")
+
+        # Also check MIME type for files without extensions or unknown extensions
+        mime_type = get_mime_type(str(path), default_mime="text/plain")
+        is_text_mime = mime_type and mime_type.startswith("text/")
+
+        if path.suffix.lower() not in self.supported_extensions and not is_text_mime:
+            # Try to detect if it's a text file by reading a small sample
+            try:
+                with open(path, "rb") as f:
+                    sample = f.read(1024)
+                    # Check if the sample contains mostly printable characters
+                    if self._is_likely_text(sample):
+                        self._color_log(f"Detected text file without standard extension: {path.suffix}", Color.yellow)
+                    else:
+                        raise ValueError(
+                            f"Unsupported file type: {path.suffix}. "
+                            f"Supported types: {', '.join(sorted(self.supported_extensions))} or text MIME types"
+                        )
+            except Exception as e:
+                raise ValueError(
+                    f"Cannot determine if file is text: {str(e)}. "
+                    f"Supported types: {', '.join(sorted(self.supported_extensions))}"
+                ) from e
+
+        return path
+
+    def _is_likely_text(self, data: bytes) -> bool:
+        """Check if binary data is likely to be text.
+
+        Args:
+            data: Binary data sample
+
+        Returns:
+            True if data appears to be text
+        """
+        if not data:
+            return True
+
+        # Check for null bytes (common in binary files)
+        if b"\x00" in data:
+            return False
+
+        # Try to decode as text
+        try:
+            data.decode("utf-8")
+            return True
+        except UnicodeDecodeError:
+            pass
+
+        # Check if most bytes are printable ASCII
+        printable_count = sum(1 for byte in data if 32 <= byte <= 126 or byte in [9, 10, 13])
+        return printable_count / len(data) > 0.7
+
+    def _detect_encoding(self, file_path: Path) -> dict[str, Any]:
+        """Detect file encoding and other characteristics.
+
+        Args:
+            file_path: Path to the text file
+
+        Returns:
+            Dictionary containing encoding information
+        """
+        encoding_info = {
+            "detected_encoding": None,
+            "confidence": 0.0,
+            "bom_detected": False,
+            "line_endings": None,
+            "is_binary": False,
+        }
+
+        try:
+            # Read file in binary mode for encoding detection
+            with open(file_path, "rb") as f:
+                raw_data = f.read()
+
+            if not raw_data:
+                encoding_info["detected_encoding"] = "utf-8"
+                encoding_info["confidence"] = 1.0
+                return encoding_info
+
+            # Check for BOM (Byte Order Mark)
+            if raw_data.startswith(b"\xef\xbb\xbf"):
+                encoding_info["bom_detected"] = True
+                encoding_info["detected_encoding"] = "utf-8-sig"
+                encoding_info["confidence"] = 1.0
+            elif raw_data.startswith(b"\xff\xfe"):
+                encoding_info["bom_detected"] = True
+                encoding_info["detected_encoding"] = "utf-16-le"
+                encoding_info["confidence"] = 1.0
+            elif raw_data.startswith(b"\xfe\xff"):
+                encoding_info["bom_detected"] = True
+                encoding_info["detected_encoding"] = "utf-16-be"
+                encoding_info["confidence"] = 1.0
+            else:
+                # Use chardet for encoding detection
+                detection_result = chardet.detect(raw_data)
+                encoding_info["detected_encoding"] = detection_result.get("encoding", "utf-8")
+                encoding_info["confidence"] = detection_result.get("confidence", 0.0)
+
+            # Detect line endings
+            if b"\r\n" in raw_data:
+                encoding_info["line_endings"] = "CRLF (Windows)"
+            elif b"\n" in raw_data:
+                encoding_info["line_endings"] = "LF (Unix/Linux/Mac)"
+            elif b"\r" in raw_data:
+                encoding_info["line_endings"] = "CR (Classic Mac)"
+            else:
+                encoding_info["line_endings"] = "None detected"
+
+            # Check if file appears to be binary
+            encoding_info["is_binary"] = not self._is_likely_text(raw_data[:1024])
+
+        except Exception as e:
+            self.logger.warning(f"Failed to detect encoding: {str(e)}")
+            encoding_info["detected_encoding"] = "utf-8"
+            encoding_info["confidence"] = 0.0
+
+        return encoding_info
+
+    def _extract_text_content(self, file_path: Path, encoding: str | None = None) -> dict[str, Any]:
+        """Extract content from text files.
+
+        Args:
+            file_path: Path to the text file
+            encoding: Specific encoding to use (None for auto-detection)
+
+        Returns:
+            Dictionary containing extracted content and metadata
+        """
+        start_time = time.time()
+
+        # Detect encoding if not specified
+        encoding_info = self._detect_encoding(file_path)
+
+        if encoding:
+            # Use specified encoding
+            target_encoding = encoding
+            self._color_log(f"Using specified encoding: {encoding}", Color.blue)
+        else:
+            # Use detected encoding
+            target_encoding = encoding_info["detected_encoding"]
+            self._color_log(
+                f"Detected encoding: {target_encoding} (confidence: {encoding_info['confidence']:.2f})", Color.blue
+            )
+
+        try:
+            # Read file content
+            with open(file_path, "r", encoding=target_encoding, errors="replace") as f:
+                content = f.read()
+
+            # Analyze content
+            lines = content.splitlines()
+
+            # Calculate statistics
+            char_count = len(content)
+            line_count = len(lines)
+            word_count = len(content.split()) if content.strip() else 0
+
+            # Find longest and shortest lines
+            line_lengths = [len(line) for line in lines]
+            max_line_length = max(line_lengths) if line_lengths else 0
+            min_line_length = min(line_lengths) if line_lengths else 0
+            avg_line_length = sum(line_lengths) / len(line_lengths) if line_lengths else 0
+
+            # Count empty lines
+            empty_lines = sum(1 for line in lines if not line.strip())
+
+            # Detect file type based on content patterns
+            content_type = self._detect_content_type(content, file_path)
+
+            processing_time = time.time() - start_time
+
+            return {
+                "content": content,
+                "encoding_info": encoding_info,
+                "statistics": {
+                    "character_count": char_count,
+                    "line_count": line_count,
+                    "word_count": word_count,
+                    "empty_lines": empty_lines,
+                    "max_line_length": max_line_length,
+                    "min_line_length": min_line_length,
+                    "avg_line_length": round(avg_line_length, 2),
+                },
+                "content_type": content_type,
+                "processing_time": processing_time,
+                "used_encoding": target_encoding,
+            }
+
+        except UnicodeDecodeError as e:
+            self.logger.error(f"Failed to decode file with encoding {target_encoding}: {str(e)}")
+            # Try with fallback encodings
+            fallback_encodings = ["utf-8", "latin-1", "cp1252", "iso-8859-1"]
+
+            for fallback_encoding in fallback_encodings:
+                if fallback_encoding != target_encoding:
+                    try:
+                        with open(file_path, "r", encoding=fallback_encoding, errors="replace") as f:
+                            content = f.read()
+
+                        self._color_log(f"Successfully read with fallback encoding: {fallback_encoding}", Color.yellow)
+
+                        # Recalculate with fallback encoding
+                        lines = content.splitlines()
+                        char_count = len(content)
+                        line_count = len(lines)
+                        word_count = len(content.split()) if content.strip() else 0
+
+                        processing_time = time.time() - start_time
+
+                        return {
+                            "content": content,
+                            "encoding_info": encoding_info,
+                            "statistics": {
+                                "character_count": char_count,
+                                "line_count": line_count,
+                                "word_count": word_count,
+                                "empty_lines": sum(1 for line in lines if not line.strip()),
+                                "max_line_length": max(len(line) for line in lines) if lines else 0,
+                                "min_line_length": min(len(line) for line in lines) if lines else 0,
+                                "avg_line_length": round(sum(len(line) for line in lines) / len(lines), 2)
+                                if lines
+                                else 0,
+                            },
+                            "content_type": self._detect_content_type(content, file_path),
+                            "processing_time": processing_time,
+                            "used_encoding": fallback_encoding,
+                            "encoding_fallback": True,
+                        }
+                    except Exception:
+                        continue
+
+            raise ValueError("Unable to decode file with any supported encoding") from e
+
+    def _detect_content_type(self, content: str, file_path: Path) -> str:
+        """Detect the type of content based on file extension and content patterns.
+
+        Args:
+            content: File content
+            file_path: Path to the file
+
+        Returns:
+            Detected content type
+        """
+        extension = file_path.suffix.lower()
+
+        # Map extensions to content types
+        extension_map = {
+            ".py": "Python source code",
+            ".js": "JavaScript source code",
+            ".html": "HTML document",
+            ".htm": "HTML document",
+            ".css": "CSS stylesheet",
+            ".json": "JSON data",
+            ".xml": "XML document",
+            ".yaml": "YAML configuration",
+            ".yml": "YAML configuration",
+            ".md": "Markdown document",
+            ".markdown": "Markdown document",
+            ".rst": "reStructuredText document",
+            ".csv": "CSV data",
+            ".tsv": "TSV data",
+            ".sql": "SQL script",
+            ".log": "Log file",
+            ".ini": "Configuration file",
+            ".cfg": "Configuration file",
+            ".conf": "Configuration file",
+        }
+
+        if extension in extension_map:
+            return extension_map[extension]
+
+        # Content-based detection
+        content_lower = content.lower().strip()
+
+        if content_lower.startswith("<?xml"):
+            return "XML document"
+        elif content_lower.startswith("{") and content_lower.endswith("}"):
+            return "JSON-like data"
+        elif content_lower.startswith("[") and content_lower.endswith("]"):
+            return "JSON array or configuration"
+        elif "#!/" in content[:50]:
+            return "Script file"
+        elif content.count(",") > content.count("\n") * 2:
+            return "CSV-like data"
+        else:
+            return "Plain text"
+
+    def _format_content_for_llm(
+        self, extraction_result: dict[str, Any], output_format: str, max_length: int | None = None
+    ) -> str:
+        """Format extracted text content to be LLM-friendly.
+
+        Args:
+            extraction_result: Result from _extract_text_content
+            output_format: Desired output format
+            max_length: Maximum length of content to include (None for no limit)
+
+        Returns:
+            Formatted content string
+        """
+        content = extraction_result["content"]
+        stats = extraction_result["statistics"]
+        content_type = extraction_result["content_type"]
+
+        # Truncate content if needed
+        if max_length and len(content) > max_length:
+            content = (
+                content[:max_length]
+                + f"\n\n[Content truncated - showing first {max_length} characters of {stats['character_count']} total]"
+            )
+
+        if output_format.lower() == "markdown":
+            formatted_parts = []
+            formatted_parts.append("# Text Document Content\n")
+            formatted_parts.append(f"**File Type:** {content_type}\n")
+            formatted_parts.append(f"**Encoding:** {extraction_result['used_encoding']}\n")
+            formatted_parts.append("**Statistics:**\n")
+            formatted_parts.append(f"- Characters: {stats['character_count']:,}\n")
+            formatted_parts.append(f"- Lines: {stats['line_count']:,}\n")
+            formatted_parts.append(f"- Words: {stats['word_count']:,}\n")
+            formatted_parts.append(f"- Empty lines: {stats['empty_lines']:,}\n")
+            formatted_parts.append(f"- Average line length: {stats['avg_line_length']} characters\n\n")
+
+            formatted_parts.append(f"## Content\n\n```\n{content}\n```")
+
+            return "".join(formatted_parts)
+
+        elif output_format.lower() == "json":
+            json_data = {
+                "document_info": {
+                    "content_type": content_type,
+                    "encoding": extraction_result["used_encoding"],
+                    "statistics": stats,
+                },
+                "content": content,
+            }
+
+            return json.dumps(json_data, indent=2, ensure_ascii=False)
+
+        elif output_format.lower() == "html":
+            html_parts = []
+            html_parts.append("<html><head><meta charset='utf-8'></head><body>")
+            html_parts.append("<h1>Text Document Content</h1>")
+            html_parts.append(f"<p><strong>File Type:</strong> {content_type}</p>")
+            html_parts.append(f"<p><strong>Encoding:</strong> {extraction_result['used_encoding']}</p>")
+            html_parts.append("<h2>Statistics</h2>")
+            html_parts.append("<ul>")
+            html_parts.append(f"<li>Characters: {stats['character_count']:,}</li>")
+            html_parts.append(f"<li>Lines: {stats['line_count']:,}</li>")
+            html_parts.append(f"<li>Words: {stats['word_count']:,}</li>")
+            html_parts.append(f"<li>Empty lines: {stats['empty_lines']:,}</li>")
+            html_parts.append(f"<li>Average line length: {stats['avg_line_length']} characters</li>")
+            html_parts.append("</ul>")
+            html_parts.append("<h2>Content</h2>")
+            html_parts.append(f"<pre><code>{content}</code></pre>")
+            html_parts.append("</body></html>")
+
+            return "".join(html_parts)
+
+        else:  # text format
+            text_parts = []
+            text_parts.append(f"Text Document Content\n{'=' * 50}\n")
+            text_parts.append(f"File Type: {content_type}\n")
+            text_parts.append(f"Encoding: {extraction_result['used_encoding']}\n")
+            text_parts.append("\nStatistics:\n")
+            text_parts.append(f"  Characters: {stats['character_count']:,}\n")
+            text_parts.append(f"  Lines: {stats['line_count']:,}\n")
+            text_parts.append(f"  Words: {stats['word_count']:,}\n")
+            text_parts.append(f"  Empty lines: {stats['empty_lines']:,}\n")
+            text_parts.append(f"  Average line length: {stats['avg_line_length']} characters\n")
+            text_parts.append(f"\nContent:\n{'-' * 30}\n{content}")
+
+            return "".join(text_parts)
+
+    def mcp_extract_text_content(
+        self,
+        file_path: str = Field(description="Path to the text document file to extract content from"),
+        output_format: Literal["markdown", "json", "html", "text"] = Field(
+            default="markdown", description="Output format: 'markdown', 'json', 'html', or 'text'"
+        ),
+        encoding: str | None = Field(default=None, description="Specific encoding to use (None for auto-detection)"),
+        max_content_length: int | None = Field(
+            default=None, description="Maximum length of content to include in output (None for no limit)"
+        ),
+    ) -> ActionResponse:
+        """Extract content from text documents with encoding detection and analysis.
+
+        This tool provides comprehensive text document content extraction with support for:
+        - Various text file formats (TXT, MD, CSV, JSON, XML, source code, etc.)
+        - Automatic encoding detection with fallback options
+        - Content type detection and analysis
+        - Comprehensive text statistics
+        - LLM-optimized output formatting
+        - Binary file detection and handling
+
+        Args:
+            file_path: Path to the text file
+            output_format: Desired output format
+            encoding: Specific encoding to use
+            max_content_length: Maximum content length to include
+
+        Returns:
+            ActionResponse with extracted content, metadata, and file analysis
+        """
+        try:
+            # Handle FieldInfo objects
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+            if isinstance(output_format, FieldInfo):
+                output_format = output_format.default
+            if isinstance(encoding, FieldInfo):
+                encoding = encoding.default
+            if isinstance(max_content_length, FieldInfo):
+                max_content_length = max_content_length.default
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Processing text document: {file_path.name}", Color.cyan)
+
+            # Extract content from text file
+            extraction_result = self._extract_text_content(file_path, encoding)
+
+            # Check if file appears to be binary
+            if extraction_result["encoding_info"]["is_binary"]:
+                self._color_log("Warning: File appears to contain binary data", Color.yellow)
+
+            # Format content for LLM consumption
+            formatted_content = self._format_content_for_llm(extraction_result, output_format, max_content_length)
+
+            # Prepare metadata
+            file_stats = file_path.stat()
+
+            # Create text-specific metadata
+            text_metadata = {
+                "content_type": extraction_result["content_type"],
+                "encoding_info": extraction_result["encoding_info"],
+                "text_statistics": extraction_result["statistics"],
+                "used_encoding": extraction_result["used_encoding"],
+                "encoding_fallback": extraction_result.get("encoding_fallback", False),
+                "content_truncated": max_content_length and len(extraction_result["content"]) > max_content_length,
+                "original_content_length": extraction_result["statistics"]["character_count"],
+            }
+
+            document_metadata = DocumentMetadata(
+                file_name=file_path.name,
+                file_size=file_stats.st_size,
+                file_type=file_path.suffix.lower() or ".txt",
+                absolute_path=str(file_path.absolute()),
+                page_count=extraction_result["statistics"]["line_count"],  # Use line count as "page" count
+                processing_time=extraction_result["processing_time"],
+                extracted_images=[],  # Text files don't contain images
+                extracted_media=[],  # Text files don't contain media
+                output_format=output_format,
+                llm_enhanced=False,
+                ocr_applied=False,
+            )
+
+            # Combine standard and text-specific metadata
+            combined_metadata = document_metadata.model_dump()
+            combined_metadata.update(text_metadata)
+
+            self._color_log(
+                f"Successfully extracted content from {file_path.name} "
+                f"({extraction_result['statistics']['character_count']:,} characters, "
+                f"{extraction_result['statistics']['line_count']:,} lines, "
+                f"encoding: {extraction_result['used_encoding']})",
+                Color.green,
+            )
+
+            return ActionResponse(success=True, message=formatted_content, metadata=combined_metadata)
+
+        except FileNotFoundError as e:
+            self.logger.error(f"File not found: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False, message=f"File not found: {str(e)}", metadata={"error_type": "file_not_found"}
+            )
+        except ValueError as e:
+            self.logger.error(f"Invalid input: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"Invalid input: {str(e)}",
+                metadata={"error_type": "invalid_input"},
+            )
+        except Exception as e:
+            self.logger.error(f"Text extraction failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"Text extraction failed: {str(e)}",
+                metadata={"error_type": "extraction_error"},
+            )
+
+    def mcp_list_supported_formats(self) -> ActionResponse:
+        """List all supported text formats for extraction.
+
+        Returns:
+            ActionResponse with list of supported file formats and their descriptions
+        """
+        supported_formats = {
+            "TXT": "Plain text files (.txt, .text)",
+            "Markdown": "Markdown documents (.md, .markdown)",
+            "CSV/TSV": "Comma/Tab separated values (.csv, .tsv)",
+            "JSON": "JSON data files (.json)",
+            "XML": "XML documents (.xml)",
+            "YAML": "YAML configuration files (.yaml, .yml)",
+            "Source Code": "Programming language files (.py, .js, .html, .css, etc.)",
+            "Configuration": "Config files (.ini, .cfg, .conf, .properties)",
+            "Logs": "Log files (.log)",
+            "Documentation": "Documentation files (.rst, .rtf)",
+            "Scripts": "Script files (.sh, .bat, .ps1)",
+            "Other Text": "Any file with text MIME type or detectable text content",
+        }
+
+        format_list = "\n".join(
+            [f"**{format_name}**: {description}" for format_name, description in supported_formats.items()]
+        )
+
+        return ActionResponse(
+            success=True,
+            message=f"Supported text formats:\n\n{format_list}\n\n"
+            "**Note:** The service automatically detects encoding and "
+            "can handle files without standard extensions if they contain readable text.",
+            metadata={
+                "supported_formats": list(supported_formats.keys()),
+                "total_formats": len(supported_formats),
+                "encoding_detection": True,
+                "binary_detection": True,
+            },
+        )
+
+
+# Example usage and entry point
+if __name__ == "__main__":
+    load_dotenv()
+
+    # Default arguments for testing
+    args = ActionArguments(
+        name="text_extraction_service",
+        transport="stdio",
+        workspace=os.getenv("AWORLD_WORKSPACE", "~"),
+    )
+
+    # Initialize and run the text extraction service
+    try:
+        service = TextExtractionCollection(args)
+        service.run()
+    except Exception as e:
+        print(f"An error occurred: {e}: {traceback.format_exc()}")