ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter9/gaia-experience/AWorld/examples/gaia/mcp_collections/media/audio.py

@@ -0,0 +1,555 @@
+import json
+import os
+import subprocess
+import time
+import traceback
+from pathlib import Path
+from typing import Any, Literal
+
+from dotenv import load_dotenv
+from openai import OpenAI
+from pydantic import BaseModel, Field
+from pydantic.fields import FieldInfo
+
+from aworld.logs.util import Color
+from examples.gaia.mcp_collections.base import ActionArguments, ActionCollection, ActionResponse
+
+
+class AudioMetadata(BaseModel):
+    """Metadata extracted from audio processing."""
+
+    file_name: str = Field(description="Original audio file name")
+    file_size: int = Field(description="File size in bytes")
+    file_type: str = Field(description="Audio file type/extension")
+    absolute_path: str = Field(description="Absolute path to the audio file")
+    duration: float | None = Field(default=None, description="Duration of audio in seconds")
+    sample_rate: int | None = Field(default=None, description="Audio sample rate in Hz")
+    channels: int | None = Field(default=None, description="Number of audio channels")
+    bitrate: int | None = Field(default=None, description="Audio bitrate in kbps")
+    codec: str | None = Field(default=None, description="Audio codec used")
+    processing_time: float = Field(description="Time taken to process the audio in seconds")
+    output_files: list[str] = Field(default_factory=list, description="Paths to generated output files")
+    transcription: str | None = Field(default=None, description="Transcribed text from audio")
+    word_count: int | None = Field(default=None, description="Number of words in transcription")
+    output_format: str = Field(description="Format of the processed output")
+
+
+class AudioCollection(ActionCollection):
+    """MCP service for comprehensive audio processing using ffmpeg.
+
+    Supports various audio operations including:
+    - Audio format conversion
+    - Audio transcription (speech-to-text)
+    - Audio metadata extraction
+    - Audio quality enhancement
+    - Audio trimming and editing
+    - Audio analysis and feature extraction
+    """
+
+    def __init__(self, arguments: ActionArguments) -> None:
+        super().__init__(arguments)
+        self._audio_output_dir = self.workspace / "processed_audio"
+        self._audio_output_dir.mkdir(exist_ok=True)
+
+        # Supported audio formats
+        self.supported_extensions = {
+            ".mp3",
+            ".wav",
+            ".flac",
+            ".aac",
+            ".ogg",
+            ".m4a",
+            ".wma",
+            ".opus",
+            ".aiff",
+            ".au",
+            ".ra",
+            ".amr",
+        }
+
+        self._color_log("Audio Processing Service initialized", Color.green, "debug")
+        self._color_log(f"Audio output directory: {self._audio_output_dir}", Color.blue, "debug")
+
+        # Check ffmpeg availability
+        self._check_ffmpeg_availability()
+
+    def _check_ffmpeg_availability(self) -> bool:
+        """Check if ffmpeg is available in the system.
+
+        Returns:
+            bool: True if ffmpeg is available, False otherwise
+        """
+        try:
+            result = subprocess.run(["ffmpeg", "-version"], capture_output=True, text=True, timeout=10, check=False)
+            if result.returncode == 0:
+                self._color_log("FFmpeg is available", Color.green, "debug")
+            else:
+                self._color_log("FFmpeg not found in system PATH", Color.red)
+            return result.returncode == 0
+        except (subprocess.TimeoutExpired, FileNotFoundError):
+            self._color_log(
+                "FFmpeg not available or timeout, Please refer to https://github.com/inclusionAI/AWorld/blob/main/examples/gaia/README.md#system-tools-setup for more details",
+                Color.red,
+            )
+            return False
+
+    def _prepare_audio_for_transcription(self, file_path: Path) -> Path:
+        """Prepare audio file for transcription by converting to optimal format.
+
+        Args:
+            file_path: Path to the original audio file
+
+        Returns:
+            Path to the prepared audio file (WAV format, 16kHz)
+        """
+        output_path = self._audio_output_dir / f"{file_path.stem}_for_transcription.wav"
+
+        # Convert to WAV format with 16kHz sample rate for optimal transcription
+        cmd = [
+            "ffmpeg",
+            "-i",
+            str(file_path),
+            "-ar",
+            "16000",  # 16kHz sample rate
+            "-ac",
+            "1",  # Mono channel
+            "-c:a",
+            "pcm_s16le",  # 16-bit PCM
+            "-y",
+            str(output_path),
+        ]
+
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=300, check=False)
+
+        if result.returncode != 0:
+            raise RuntimeError(f"Audio preparation for transcription failed: {result.stderr}")
+
+        return output_path
+
+    def _transcribe_with_whisper(self, audio_path: Path) -> dict[str, Any]:
+        """Transcribe audio using OpenAI Whisper.
+
+        Args:
+            audio_path: Path to the audio file
+
+        Returns:
+            Dictionary containing transcription results
+        """
+        try:
+            client: OpenAI = OpenAI(api_key=os.getenv("AUDIO_LLM_API_KEY"), base_url=os.getenv("AUDIO_LLM_BASE_URL"))
+
+            # Use the file for transcription
+            with open(audio_path, "rb") as audio_file:
+                transcription: str = client.audio.transcriptions.create(
+                    file=audio_file,
+                    model=os.getenv("AUDIO_LLM_MODEL_NAME"),
+                    response_format="text",
+                )
+
+            return {"text": transcription.strip() if transcription else ""}
+        except Exception as e:
+            raise RuntimeError(f"Audio transcription failed: {e}: {traceback.format_exc()}") from e
+
+    def _get_audio_metadata(self, file_path: Path) -> dict[str, Any]:
+        """Extract audio metadata using ffprobe.
+
+        Args:
+            file_path: Path to the audio file
+
+        Returns:
+            Dictionary containing audio metadata
+        """
+        try:
+            cmd = ["ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", str(file_path)]
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=30, check=False)
+
+            if result.returncode == 0:
+                metadata = json.loads(result.stdout)
+
+                # Extract relevant audio information
+                format_info = metadata.get("format", {})
+                streams = metadata.get("streams", [])
+                audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), {})
+
+                return {
+                    "duration": float(format_info.get("duration", 0)),
+                    "sample_rate": int(audio_stream.get("sample_rate", 0)) if audio_stream.get("sample_rate") else None,
+                    "channels": int(audio_stream.get("channels", 0)) if audio_stream.get("channels") else None,
+                    "bitrate": int(format_info.get("bit_rate", 0)) // 1000 if format_info.get("bit_rate") else None,
+                    "codec": audio_stream.get("codec_name"),
+                }
+            else:
+                self.logger.warning(f"Failed to extract metadata: {result.stderr}")
+                return {}
+
+        except Exception as e:
+            self.logger.error(f"Error extracting audio metadata: {str(e)}")
+            return {}
+
+    def _trim_audio(self, input_path: Path, start_time: float, duration: float | None = None) -> Path:
+        """Trim audio file to specified time range.
+
+        Args:
+            input_path: Path to input audio file
+            start_time: Start time in seconds
+            duration: Duration in seconds (if None, trim to end)
+
+        Returns:
+            Path to trimmed audio file
+        """
+        output_path = self._audio_output_dir / f"{input_path.stem}_trimmed{input_path.suffix}"
+
+        cmd = ["ffmpeg", "-i", str(input_path), "-ss", str(start_time), "-y"]
+
+        if duration is not None:
+            cmd.extend(["-t", str(duration)])
+
+        cmd.extend(["-c", "copy", str(output_path)])
+
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=300, check=False)
+
+        if result.returncode != 0:
+            raise RuntimeError(f"Audio trimming failed: {result.stderr}")
+
+        return output_path
+
+    def mcp_transcribe_audio(
+        self,
+        file_path: str = Field(description="Path to the audio file to transcribe"),
+        model_size: Literal["tiny", "base", "small", "medium", "large"] = Field(
+            default="base",
+            description="Whisper model size: tiny (fastest), base (balanced), small, medium, large (most accurate)",
+        ),
+        output_format: Literal["text", "detailed", "segments"] = Field(
+            default="text",
+            description="Output format: 'text' (plain text), 'detailed' (with metadata), 'segments' (timestamped)",
+        ),
+    ) -> ActionResponse:
+        """Transcribe audio file to text using OpenAI Whisper.
+
+        This tool converts speech in audio files to text with high accuracy.
+        Supports multiple languages and provides various output formats including
+        timestamped segments for detailed analysis.
+
+        Args:
+            file_path: Path to the audio file to transcribe
+            model_size: Whisper model size affecting speed vs accuracy trade-off
+            output_format: Format of transcription output
+
+        Returns:
+            ActionResponse with transcribed text and detailed metadata
+        """
+        try:
+            # Handle FieldInfo objects
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+            if isinstance(model_size, FieldInfo):
+                model_size = model_size.default
+            if isinstance(output_format, FieldInfo):
+                output_format = output_format.default
+
+            start_time = time.time()
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Transcribing audio: {file_path.name}", Color.cyan)
+
+            # Get original metadata
+            original_metadata = self._get_audio_metadata(file_path)
+
+            # Prepare audio for transcription
+            prepared_audio = self._prepare_audio_for_transcription(file_path)
+
+            # Perform transcription
+            transcription_result = self._transcribe_with_whisper(prepared_audio)
+
+            processing_time = time.time() - start_time
+
+            # Prepare file statistics
+            file_stats = file_path.stat()
+
+            # Count words in transcription
+            word_count = len(transcription_result["text"].split()) if transcription_result["text"] else 0
+
+            # Create metadata object
+            audio_metadata = AudioMetadata(
+                file_name=file_path.name,
+                file_size=file_stats.st_size,
+                file_type=file_path.suffix.lower(),
+                absolute_path=str(file_path.absolute()),
+                duration=original_metadata.get("duration"),
+                sample_rate=original_metadata.get("sample_rate"),
+                channels=original_metadata.get("channels"),
+                bitrate=original_metadata.get("bitrate"),
+                codec=original_metadata.get("codec"),
+                processing_time=processing_time,
+                output_files=[str(prepared_audio)],
+                transcription=transcription_result["text"],
+                word_count=word_count,
+                output_format=f"transcription_{output_format}",
+            )
+
+            # Format output based on requested format
+            if output_format == "text":
+                result_message = transcription_result["text"]
+            elif output_format == "detailed":
+                confidence_str = (
+                    f"{transcription_result['confidence']:.2f}" if transcription_result["confidence"] else "N/A"
+                )
+                result_message = (
+                    f"Transcription Results for {file_path.name}:\n\n"
+                    f"**Text:** {transcription_result['text']}\n\n"
+                    f"**Confidence:** {confidence_str}\n"
+                    f"**Word Count:** {word_count}\n"
+                    f"**Duration:** {original_metadata.get('duration', 0):.2f} seconds\n"
+                    f"**Model:** {model_size}\n"
+                    f"**Processing Time:** {processing_time:.2f} seconds"
+                )
+            elif output_format == "segments":
+                segments_text = "\n".join(
+                    [
+                        f"[{seg.get('start', 0):.2f}s - {seg.get('end', 0):.2f}s]: {seg.get('text', '').strip()}"
+                        for seg in transcription_result.get("segments", [])
+                    ]
+                )
+                result_message = (
+                    f"Timestamped Transcription for {file_path.name}:\n\n"
+                    f"{segments_text}\n\n"
+                    f"**Full Text:** {transcription_result['text']}"
+                )
+            else:
+                result_message = transcription_result["text"]
+
+            # Clean up temporary file
+            try:
+                prepared_audio.unlink()
+            except Exception:
+                pass  # Ignore cleanup errors
+
+            self._color_log(f"Transcription completed: {word_count} words, {processing_time:.2f}s", Color.green)
+
+            return ActionResponse(success=True, message=result_message, metadata=audio_metadata.model_dump())
+
+        except Exception as e:
+            self.logger.error(f"Audio transcription failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"Audio transcription failed: {str(e)}",
+                metadata={"error_type": "transcription_error"},
+            )
+
+    def mcp_extract_audio_metadata(
+        self,
+        file_path: str = Field(description="Path to the audio file to analyze"),
+    ) -> ActionResponse:
+        """Extract comprehensive metadata from audio files.
+
+        This tool analyzes audio files and extracts detailed metadata including
+        duration, sample rate, channels, bitrate, codec, and other technical information.
+
+        Args:
+            file_path: Path to the audio file to analyze
+
+        Returns:
+            ActionResponse with detailed audio metadata
+        """
+        try:
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+
+            start_time = time.time()
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Extracting metadata from: {file_path.name}", Color.cyan)
+
+            # Extract metadata
+            metadata = self._get_audio_metadata(file_path)
+            processing_time = time.time() - start_time
+
+            # Prepare file statistics
+            file_stats = file_path.stat()
+
+            # Create metadata object
+            audio_metadata = AudioMetadata(
+                file_name=file_path.name,
+                file_size=file_stats.st_size,
+                file_type=file_path.suffix.lower(),
+                absolute_path=str(file_path.absolute()),
+                duration=metadata.get("duration"),
+                sample_rate=metadata.get("sample_rate"),
+                channels=metadata.get("channels"),
+                bitrate=metadata.get("bitrate"),
+                codec=metadata.get("codec"),
+                processing_time=processing_time,
+                output_files=[],
+                output_format="metadata",
+            )
+
+            # Format metadata for LLM consumption
+            result_message = (
+                f"Audio Metadata for {file_path.name}:\n"
+                f"Duration: {metadata.get('duration', 'Unknown'):.2f} seconds\n"
+                f"Sample Rate: {metadata.get('sample_rate', 'Unknown')} Hz\n"
+                f"Channels: {metadata.get('channels', 'Unknown')}\n"
+                f"Bitrate: {metadata.get('bitrate', 'Unknown')} kbps\n"
+                f"Codec: {metadata.get('codec', 'Unknown')}\n"
+                f"File Size: {file_stats.st_size / 1024 / 1024:.2f} MB\n"
+                f"Format: {file_path.suffix.upper()}"
+            )
+
+            self._color_log(f"Metadata extraction completed in {processing_time:.2f}s", Color.green)
+
+            return ActionResponse(success=True, message=result_message, metadata=audio_metadata.model_dump())
+
+        except Exception as e:
+            self.logger.error(f"Metadata extraction failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"Metadata extraction failed: {str(e)}",
+                metadata={"error_type": "metadata_error"},
+            )
+
+    def mcp_trim_audio(
+        self,
+        file_path: str = Field(description="Path to the audio file to trim"),
+        start_time: float = Field(description="Start time in seconds"),
+        duration: float | None = Field(default=None, description="Duration in seconds (if None, trim to end)"),
+    ) -> ActionResponse:
+        """Trim audio file to specified time range.
+
+        This tool cuts audio files to extract specific segments based on start time
+        and duration. Useful for creating clips or removing unwanted sections.
+
+        Args:
+            file_path: Path to the source audio file
+            start_time: Start time in seconds for trimming
+            duration: Duration of the trimmed segment (optional)
+
+        Returns:
+            ActionResponse with trimmed audio file path and metadata
+        """
+        try:
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+            if isinstance(start_time, FieldInfo):
+                start_time = start_time.default
+            if isinstance(duration, FieldInfo):
+                duration = duration.default
+
+            process_start = time.time()
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Trimming audio: {file_path.name}", Color.cyan)
+
+            # Get original metadata
+            original_metadata = self._get_audio_metadata(file_path)
+
+            # Validate time parameters
+            if start_time < 0:
+                raise ValueError("Start time cannot be negative")
+            if duration is not None and duration <= 0:
+                raise ValueError("Duration must be positive")
+            if original_metadata.get("duration") and start_time >= original_metadata["duration"]:
+                raise ValueError("Start time exceeds audio duration")
+
+            # Trim audio
+            output_path = self._trim_audio(file_path, start_time, duration)
+
+            # Get trimmed file metadata
+            trimmed_metadata = self._get_audio_metadata(output_path)
+            processing_time = time.time() - process_start
+
+            # Prepare file statistics
+            file_stats = file_path.stat()
+
+            # Create metadata object
+            audio_metadata = AudioMetadata(
+                file_name=file_path.name,
+                file_size=file_stats.st_size,
+                file_type=file_path.suffix.lower(),
+                absolute_path=str(file_path.absolute()),
+                duration=trimmed_metadata.get("duration"),
+                sample_rate=trimmed_metadata.get("sample_rate"),
+                channels=trimmed_metadata.get("channels"),
+                bitrate=trimmed_metadata.get("bitrate"),
+                codec=trimmed_metadata.get("codec"),
+                processing_time=processing_time,
+                output_files=[str(output_path)],
+                output_format="trimmed_audio",
+            )
+
+            end_time = start_time + (duration or (original_metadata.get("duration", 0) - start_time))
+            result_message = (
+                f"Successfully trimmed {file_path.name}\n"
+                f"Original duration: {original_metadata.get('duration', 0):.2f} seconds\n"
+                f"Trimmed segment: {start_time:.2f}s - {end_time:.2f}s\n"
+                f"New duration: {trimmed_metadata.get('duration', 0):.2f} seconds\n"
+                f"Output file: {output_path.name}"
+            )
+
+            self._color_log(f"Audio trimming completed in {processing_time:.2f}s", Color.green)
+
+            return ActionResponse(success=True, message=result_message, metadata=audio_metadata.model_dump())
+
+        except Exception as e:
+            self.logger.error(f"Audio trimming failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False, message=f"Audio trimming failed: {str(e)}", metadata={"error_type": "trimming_error"}
+            )
+
+    def mcp_list_supported_formats(self) -> ActionResponse:
+        """List all supported audio formats for processing.
+
+        Returns:
+            ActionResponse with list of supported audio formats and their descriptions
+        """
+        supported_formats = {
+            "MP3": "MPEG Audio Layer III (.mp3) - Most common compressed format",
+            "WAV": "Waveform Audio File Format (.wav) - Uncompressed, high quality",
+            "FLAC": "Free Lossless Audio Codec (.flac) - Lossless compression",
+            "AAC": "Advanced Audio Coding (.aac) - Efficient compression",
+            "OGG": "Ogg Vorbis (.ogg) - Open source compressed format",
+            "M4A": "MPEG-4 Audio (.m4a) - Apple's preferred format",
+            "WMA": "Windows Media Audio (.wma) - Microsoft format",
+            "OPUS": "Opus Audio (.opus) - Modern, efficient codec",
+            "AIFF": "Audio Interchange File Format (.aiff) - Apple's uncompressed format",
+            "AU": "Sun Audio (.au) - Unix audio format",
+            "RA": "RealAudio (.ra) - Streaming audio format",
+            "AMR": "Adaptive Multi-Rate (.amr) - Mobile audio format",
+        }
+
+        format_list = "\n".join(
+            [f"**{format_name}**: {description}" for format_name, description in supported_formats.items()]
+        )
+
+        return ActionResponse(
+            success=True,
+            message=f"Supported audio formats:\n\n{format_list}",
+            metadata={
+                "supported_formats": list(supported_formats.keys()),
+                "total_formats": len(supported_formats),
+                "ffmpeg_available": self._check_ffmpeg_availability(),
+            },
+        )
+
+
+# Example usage and entry point
+if __name__ == "__main__":
+    load_dotenv()
+
+    # Default arguments for testing
+    args = ActionArguments(
+        name="audio_processing_service",
+        transport="stdio",
+        workspace=os.getenv("AWORLD_WORKSPACE", "~"),
+    )
+
+    # Initialize and run the audio processing service
+    try:
+        service = AudioCollection(args)
+        service.run()
+        print("Audio processing service started")
+    except Exception as e:
+        print(f"An error occurred: {e}: {traceback.format_exc()}")

chapter9/gaia-experience/AWorld/examples/gaia/mcp_collections/media/image.py

@@ -0,0 +1,484 @@
+import base64
+import os
+import time
+import traceback
+from io import BytesIO
+from pathlib import Path
+from typing import Any
+
+import pytesseract
+from dotenv import load_dotenv
+from PIL import Image, ImageEnhance, ImageFilter
+from pydantic import BaseModel, Field
+from pydantic.fields import FieldInfo
+
+from aworld.config.conf import AgentConfig
+from aworld.logs.util import Color
+from aworld.models.llm import call_llm_model, get_llm_model
+from aworld.models.model_response import ModelResponse
+from examples.gaia.mcp_collections.base import ActionArguments, ActionCollection, ActionResponse
+
+
+class ImageMetadata(BaseModel):
+    """Metadata extracted from image processing."""
+
+    file_name: str = Field(description="Original image file name")
+    file_size: int = Field(description="File size in bytes")
+    file_type: str = Field(description="Image file type/extension")
+    absolute_path: str = Field(description="Absolute path to the image file")
+    width: int | None = Field(default=None, description="Image width in pixels")
+    height: int | None = Field(default=None, description="Image height in pixels")
+    mode: str | None = Field(default=None, description="Image color mode (RGB, RGBA, L, etc.)")
+    format: str | None = Field(default=None, description="Image format (JPEG, PNG, etc.)")
+    has_transparency: bool = Field(default=False, description="Whether image has transparency")
+    processing_time: float = Field(description="Time taken to process the image in seconds", exclude=True)
+    output_files: list[str] = Field(default_factory=list, description="Paths to generated output files")
+    extracted_text: str | None = Field(default=None, description="Text extracted via OCR")
+    analysis_result: str | None = Field(default=None, description="AI analysis result")
+    compression_ratio: float | None = Field(default=None, description="Compression ratio if optimized")
+    output_format: str = Field(description="Format of the processed output")
+
+
+class ImageCollection(ActionCollection):
+    """MCP service for comprehensive image processing and analysis.
+
+    Supports various image operations including:
+    - Metadata extraction
+    - OCR (Optical Character Recognition)
+    - AI-powered image analysis and reasoning
+    """
+
+    def __init__(self, arguments: ActionArguments) -> None:
+        super().__init__(arguments)
+        self._image_output_dir = self.workspace / "processed_images"
+        self._image_output_dir.mkdir(exist_ok=True)
+
+        # Supported image formats
+        self.supported_extensions = {
+            ".jpg",
+            ".jpeg",
+            ".png",
+            ".gif",
+            ".webp",
+            ".bmp",
+            ".tiff",
+            ".tif",
+            ".ico",
+            ".svg",
+        }
+
+        self._llm_config = AgentConfig(
+            llm_provider="openai",
+            llm_model_name=os.getenv("IMAGE_LLM_MODEL_NAME", "gpt-4o"),
+            llm_api_key=os.getenv("IMAGE_LLM_API_KEY"),
+            llm_base_url=os.getenv("IMAGE_LLM_BASE_URL"),
+        )
+
+        self._color_log("Image Processing Service initialized", Color.green, "debug")
+        self._color_log(f"Image output directory: {self._image_output_dir}", Color.blue, "debug")
+
+    def _load_image(self, file_path: Path) -> Image.Image:
+        """Load image from file path.
+
+        Args:
+            file_path: Path to the image file
+
+        Returns:
+            PIL Image object
+        """
+        try:
+            return Image.open(file_path)
+        except Exception as e:
+            raise RuntimeError(f"Failed to load image {file_path}: {str(e)}") from e
+
+    def _get_image_metadata(self, image: Image.Image, file_path: Path) -> dict[str, Any]:
+        """Extract metadata from PIL Image object.
+
+        Args:
+            image: PIL Image object
+            file_path: Path to the original file
+
+        Returns:
+            Dictionary containing image metadata
+        """
+        return {
+            "width": image.width,
+            "height": image.height,
+            "mode": image.mode,
+            "format": image.format or file_path.suffix.upper().lstrip("."),
+            "has_transparency": image.mode in ("RGBA", "LA") or "transparency" in image.info,
+        }
+
+    def _optimize_image(self, image: Image.Image, max_size: tuple[int, int] | None = None) -> Image.Image:
+        """Optimize image for size and quality.
+
+        Args:
+            image: PIL Image object
+            max_size: Maximum dimensions (width, height)
+
+        Returns:
+            Optimized PIL Image object
+        """
+        optimized = image.copy()
+
+        # Resize if max_size specified
+        if max_size:
+            optimized.thumbnail(max_size, Image.Resampling.LANCZOS)
+
+        return optimized
+
+    def _perform_ocr(self, image: Image.Image) -> str:
+        """Perform OCR on image to extract text.
+
+        Args:
+            image: PIL Image object
+
+        Returns:
+            Extracted text string
+        """
+        try:
+            return pytesseract.image_to_string(image).strip()
+        except ImportError:
+            return "OCR not available - pytesseract not installed"
+        except Exception as e:
+            return f"OCR failed: {str(e)}"
+
+    def _analyze_with_ai(self, image_base64: str, task: str) -> str:
+        """Analyze image using AI model.
+
+        Args:
+            image_base64: Base64 encoded image
+            task: Analysis task description
+
+        Returns:
+            AI analysis result
+        """
+        try:
+            messages = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"role": "text", "content": task},
+                        {"type": "image_url", "image_url": {"url": image_base64}},
+                    ],
+                },
+            ]
+            response: ModelResponse = call_llm_model(
+                llm_model=get_llm_model(conf=self._llm_config),
+                messages=messages,
+                temperature=float(os.getenv("LLM_TEMPERATURE", "1.0")),
+            )
+            self._color_log(f"{response.content=}", Color.green)
+            return response.content
+        except Exception as e:
+            return f"AI analysis failed: {str(e)}"
+
+    def _image_to_base64(self, image: Image.Image, output_format: str = "JPEG") -> str:
+        """Convert PIL Image to base64 string.
+
+        Args:
+            image: PIL Image object
+            output_format: Output format (JPEG, PNG, etc.)
+
+        Returns:
+            Base64 encoded image string
+        """
+        buffer = BytesIO()
+
+        # Convert RGBA or P to RGB for JPEG
+        if output_format.upper() == "JPEG":
+            if image.mode in ("RGBA", "LA"):
+                background = Image.new("RGB", image.size, (255, 255, 255))
+                background.paste(image, mask=image.split()[-1] if image.mode == "RGBA" else None)
+                image = background
+            elif image.mode == "P":
+                image = image.convert("RGB")
+
+        image.save(buffer, format=output_format, quality=85 if output_format.upper() == "JPEG" else None)
+
+        mime_type = f"image/{output_format.lower()}"
+        img_base64 = base64.b64encode(buffer.getvalue()).decode()
+
+        return f"data:{mime_type};base64,{img_base64}"
+
+    def mcp_extract_text_ocr(
+        self,
+        file_path: str = Field(description="Path to the image file for OCR"),
+        language: str = Field(default="eng", description="OCR language code (e.g., 'eng', 'spa', 'fra')"),
+        preprocess: bool = Field(default=True, description="Whether to preprocess image for better OCR"),
+    ) -> ActionResponse:
+        """Extract text from images using Optical Character Recognition (OCR).
+
+        This tool uses Tesseract OCR to extract text content from images,
+        with optional preprocessing to improve recognition accuracy.
+
+        Args:
+            file_path: Path to the image file
+            language: OCR language for better recognition
+            preprocess: Whether to enhance image for OCR
+
+        Returns:
+            ActionResponse with extracted text and metadata
+        """
+        try:
+            # Handle FieldInfo objects
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+            if isinstance(language, FieldInfo):
+                language = language.default
+            if isinstance(preprocess, FieldInfo):
+                preprocess = preprocess.default
+
+            start_time = time.time()
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Performing OCR on: {file_path.name}", Color.cyan)
+
+            # Load image
+            image = self._load_image(file_path)
+            original_metadata = self._get_image_metadata(image, file_path)
+
+            # Preprocess image for better OCR if requested
+            if preprocess:
+                # Convert to grayscale
+                if image.mode != "L":
+                    image = image.convert("L")
+
+                # Enhance contrast
+                enhancer = ImageEnhance.Contrast(image)
+                image = enhancer.enhance(2.0)
+
+                # Apply slight sharpening
+                image = image.filter(ImageFilter.SHARPEN)
+
+            # Perform OCR
+            extracted_text = self._perform_ocr(image)
+            processing_time = time.time() - start_time
+
+            # Count words and characters
+            word_count = len(extracted_text.split()) if extracted_text else 0
+            char_count = len(extracted_text) if extracted_text else 0
+
+            # Create metadata object
+            image_metadata = ImageMetadata(
+                file_name=file_path.name,
+                file_size=file_path.stat().st_size,
+                file_type=file_path.suffix.lower(),
+                absolute_path=str(file_path.absolute()),
+                width=original_metadata["width"],
+                height=original_metadata["height"],
+                mode=original_metadata["mode"],
+                format=original_metadata["format"],
+                has_transparency=original_metadata["has_transparency"],
+                processing_time=processing_time,
+                output_files=[],
+                extracted_text=extracted_text,
+                output_format="ocr_text",
+            )
+
+            if extracted_text:
+                result_message = (
+                    f"OCR Results for {file_path.name}:\n\n"
+                    f"**Extracted Text:**\n{extracted_text}\n\n"
+                    f"**Statistics:**\n"
+                    f"- Words: {word_count}\n"
+                    f"- Characters: {char_count}\n"
+                    f"- Language: {language}\n"
+                    f"- Processing time: {processing_time:.2f}s"
+                )
+            else:
+                result_message = (
+                    f"No text detected in {file_path.name}."
+                    " The image may not contain readable text or OCR preprocessing may be needed."
+                )
+
+            self._color_log(f"OCR completed: {word_count} words extracted in {processing_time:.2f}s", Color.green)
+
+            return ActionResponse(success=True, message=result_message, metadata=image_metadata.model_dump())
+
+        except Exception as e:
+            self.logger.error(f"OCR failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"OCR failed: {str(e)}",
+                metadata={"error_type": "ocr_error"},
+            )
+
+    def mcp_analyze_image_ai(
+        self,
+        file_path: str = Field(description="Path to the image file for AI analysis"),
+        task: str = Field(
+            default="Describe what you see in this image",
+            description="Specific analysis task or question about the image",
+        ),
+    ) -> ActionResponse:
+        """Analyze image content using AI vision models.
+
+        This tool uses advanced AI models to analyze and describe image content,
+        answer questions about images, or perform specific visual reasoning tasks.
+
+        Args:
+            file_path: Path to the image file
+            task: Specific analysis task or question
+
+        Returns:
+            ActionResponse with AI analysis results and metadata
+        """
+        try:
+            # Handle FieldInfo objects
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+            if isinstance(task, FieldInfo):
+                task = task.default
+
+            start_time = time.time()
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Analyzing image with AI: {file_path.name}", Color.cyan)
+
+            # Load image
+            image = self._load_image(file_path)
+            original_metadata = self._get_image_metadata(image, file_path)
+
+            # Convert to base64 for AI analysis
+            image_base64 = self._image_to_base64(image, "JPEG")
+
+            # Perform AI analysis
+            analysis_result = self._analyze_with_ai(image_base64, task)
+            processing_time = time.time() - start_time
+
+            # Create metadata object
+            metadata_dict = {
+                "file_name": file_path.name,
+                "file_size": file_path.stat().st_size,
+                "file_type": file_path.suffix.lower(),
+                "absolute_path": str(file_path.absolute()),
+                "width": original_metadata["width"],
+                "height": original_metadata["height"],
+                "mode": original_metadata["mode"],
+                "format": original_metadata["format"],
+                "has_transparency": original_metadata["has_transparency"],
+                "processing_time": processing_time,
+                "output_files": [],
+                "analysis_result": analysis_result,
+                "output_format": "ai_analysis",
+            }
+
+            image_metadata = ImageMetadata(**metadata_dict)
+
+            result_message = (
+                f"AI Analysis Results for {file_path.name}:\n\n"
+                f"**Task:** {task}\n\n"
+                f"**Analysis:**\n{analysis_result}\n\n"
+                f"**Image Info:**\n"
+                f"- Dimensions: {original_metadata['width']}x{original_metadata['height']}\n"
+                f"- Format: {original_metadata['format']}\n"
+                f"- Processing time: {processing_time:.2f}s"
+            )
+
+            self._color_log(f"AI analysis completed in {processing_time:.2f}s", Color.green)
+
+            return ActionResponse(success=True, message=result_message, metadata=image_metadata.model_dump())
+
+        except Exception as e:
+            self.logger.error(f"AI image analysis failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"AI image analysis failed: {str(e)}",
+                metadata={"error_type": "ai_analysis_error"},
+            )
+
+    def mcp_get_image_metadata(
+        self,
+        file_path: str = Field(description="Path to the image file to analyze"),
+    ) -> ActionResponse:
+        """Extract comprehensive metadata from image files.
+
+        This tool analyzes image files and extracts detailed metadata including
+        dimensions, format, color mode, file size, and other technical information.
+
+        Args:
+            file_path: Path to the image file to analyze
+
+        Returns:
+            ActionResponse with detailed image metadata
+        """
+        try:
+            if isinstance(file_path, FieldInfo):
+                file_path = file_path.default
+
+            start_time = time.time()
+
+            # Validate input file
+            file_path: Path = self._validate_file_path(file_path)
+            self._color_log(f"Extracting metadata from: {file_path.name}", Color.cyan)
+
+            # Load image
+            image = self._load_image(file_path)
+            metadata = self._get_image_metadata(image, file_path)
+            processing_time = time.time() - start_time
+
+            # Get file statistics
+            file_stats = file_path.stat()
+
+            # Create metadata object
+            image_metadata = ImageMetadata(
+                file_name=file_path.name,
+                file_size=file_stats.st_size,
+                file_type=file_path.suffix.lower(),
+                absolute_path=str(file_path.absolute()),
+                width=metadata["width"],
+                height=metadata["height"],
+                mode=metadata["mode"],
+                format=metadata["format"],
+                has_transparency=metadata["has_transparency"],
+                processing_time=processing_time,
+                output_files=[],
+                output_format="metadata",
+            )
+
+            # Calculate additional info
+            aspect_ratio = metadata["width"] / metadata["height"] if metadata["height"] > 0 else 0
+            megapixels = (metadata["width"] * metadata["height"]) / 1_000_000
+
+            result_message = (
+                f"Image Metadata for {file_path.name}:\n"
+                f"Dimensions: {metadata['width']}x{metadata['height']} pixels\n"
+                f"Aspect Ratio: {aspect_ratio:.2f}:1\n"
+                f"Megapixels: {megapixels:.2f} MP\n"
+                f"Color Mode: {metadata['mode']}\n"
+                f"Format: {metadata['format']}\n"
+                f"Has Transparency: {metadata['has_transparency']}\n"
+                f"File Size: {file_stats.st_size / 1024 / 1024:.2f} MB\n"
+                f"File Type: {file_path.suffix.upper()}"
+            )
+
+            self._color_log(f"Metadata extraction completed in {processing_time:.2f}s", Color.green)
+
+            return ActionResponse(success=True, message=result_message, metadata=image_metadata.model_dump())
+
+        except Exception as e:
+            self.logger.error(f"Metadata extraction failed: {str(e)}: {traceback.format_exc()}")
+            return ActionResponse(
+                success=False,
+                message=f"Metadata extraction failed: {str(e)}",
+                metadata={"error_type": "metadata_error"},
+            )
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    # Default arguments for testing
+    args = ActionArguments(
+        name="image_analysis_service",
+        transport="stdio",
+        workspace=os.getenv("AWORLD_WORKSPACE", "~"),
+    )
+    # Initialize and run the image analysis service
+    try:
+        service = ImageCollection(args)
+        service.run()
+    except Exception as e:
+        print(f"An error occurred: {str(e)}")

chapter9/gaia-experience/AWorld/examples/gaia/mcp_collections/media/video.py

@@ -0,0 +1,992 @@
+"""
+Video MCP Service
+
+This module provides MCP service functionality for video operations including:
+- Video content analysis with AI-powered insights
+- Video summarization and key point extraction
+- Keyframe extraction with scene detection
+- Subtitle extraction from video content
+
+It handles various video formats with proper validation, error handling,
+and progress tracking while providing LLM-friendly formatted results.
+"""
+
+import base64
+import os
+import time
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+import cv2
+import numpy as np
+from dotenv import load_dotenv
+from pydantic import BaseModel, Field
+
+from aworld.config.conf import AgentConfig
+from aworld.logs.util import Color
+from aworld.models.llm import call_llm_model, get_llm_model
+from aworld.models.model_response import ModelResponse
+from examples.gaia.mcp_collections.base import (
+    ActionArguments,
+    ActionCollection,
+    ActionResponse,
+)
+
+from ..utils import get_file_from_source
+
+
+class VideoAnalysisResult(BaseModel):
+    """Video analysis result model with structured data"""
+
+    video_source: str
+    analysis_result: str
+    frame_count: int
+    duration_analyzed: float
+    success: bool
+    error: str | None = None
+
+
+class VideoSummaryResult(BaseModel):
+    """Video summary result model with structured data"""
+
+    video_source: str
+    summary: str
+    frame_count: int
+    duration_analyzed: float
+    success: bool
+    error: str | None = None
+
+
+class KeyframeResult(BaseModel):
+    """Keyframe extraction result model with file information"""
+
+    frame_paths: list[str]
+    frame_timestamps: list[float]
+    output_directory: str
+    frame_count: int
+    target_time: float
+    window_size: float
+    success: bool
+    error: str | None = None
+
+
+class VideoMetadata(BaseModel):
+    """Metadata for video operation results"""
+
+    operation: str
+    video_source: str | None = None
+    sample_rate: int | None = None
+    start_time: float | None = None
+    end_time: float | None = None
+    target_time: float | None = None
+    window_size: float | None = None
+    output_directory: str | None = None
+    frame_count: int | None = None
+    execution_time: float | None = None
+    error_type: str | None = None
+
+
+class VideoCollection(ActionCollection):
+    """MCP service for video operations with AI-powered analysis.
+
+    Provides video processing capabilities including:
+    - AI-powered video content analysis
+    - Video summarization with key insights
+    - Keyframe extraction with scene detection
+    - Subtitle extraction from video content
+    - LLM-friendly result formatting
+    - Error handling and logging
+    """
+
+    def __init__(self, arguments: ActionArguments) -> None:
+        super().__init__(arguments)
+
+        # Initialize supported video extensions
+        self.supported_extensions = {".mp4", ".avi", ".mov", ".mkv", ".webm", ".flv"}
+
+        # Video analysis prompts
+        self.video_analyze_prompt = (
+            "Input is a sequence of video frames. Given user's task: {task}. "
+            "analyze the video content following these steps:\n"
+            "1. Temporal sequence understanding\n"
+            "2. Motion and action analysis\n"
+            "3. Scene context interpretation\n"
+            "4. Object and person tracking\n"
+        )
+
+        self.video_summarize_prompt = (
+            "Input is a sequence of video frames. "
+            "Summarize the main content of the video. "
+            "Include key points, main topics, and important visual elements. "
+        )
+
+        self._color_log("Video service initialized", Color.green, "debug")
+
+    def _get_video_frames(
+        self,
+        video_source: str,
+        sample_rate: int = 2,
+        start_time: float = 0,
+        end_time: float | None = None,
+    ) -> list[dict[str, any]]:
+        """Extract frames from video with given sample rate.
+
+        Args:
+            video_source: Path or URL to the video file
+            sample_rate: Number of frames to sample per second
+            start_time: Start time of the video segment in seconds
+            end_time: End time of the video segment in seconds
+
+        Returns:
+            List of dictionaries containing frame data and timestamp
+
+        Raises:
+            ValueError: When video file cannot be opened or is not valid
+        """
+        try:
+            # Get file with validation (only video files allowed)
+            file_path, _, _ = get_file_from_source(
+                video_source,
+                max_size_mb=2500.0,  # 2500MB limit for videos
+            )
+
+            # Open video file
+            video = cv2.VideoCapture(file_path)  # pylint: disable=E1101
+            if not video.isOpened():
+                raise ValueError(f"Could not open video file: {file_path}")
+
+            fps = video.get(cv2.CAP_PROP_FPS)  # pylint: disable=E1101
+            frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT))  # pylint: disable=E1101
+            video_duration = frame_count / fps
+
+            if end_time is None:
+                end_time = video_duration
+
+            if start_time > end_time:
+                raise ValueError("Start time cannot be greater than end time.")
+
+            if start_time < 0:
+                start_time = 0
+
+            if end_time > video_duration:
+                end_time = video_duration
+
+            start_frame = int(start_time * fps)
+            end_frame = int(end_time * fps)
+
+            all_frames = []
+            frames = []
+
+            # Calculate frame interval based on sample rate
+            frame_interval = max(1, int(fps / sample_rate))
+
+            # Set the video capture to the start frame
+            video.set(cv2.CAP_PROP_POS_FRAMES, start_frame)  # pylint: disable=E1101
+
+            for i in range(start_frame, end_frame):
+                ret, frame = video.read()
+                if not ret:
+                    break
+
+                # Convert frame to JPEG format
+                _, buffer = cv2.imencode(".jpg", frame)  # pylint: disable=E1101
+                frame_data = base64.b64encode(buffer).decode("utf-8")
+
+                # Add data URL prefix for JPEG image
+                frame_data = f"data:image/jpeg;base64,{frame_data}"
+
+                all_frames.append({"data": frame_data, "time": i / fps})
+
+            for i in range(0, len(all_frames), frame_interval):
+                frames.append(all_frames[i])
+
+            video.release()
+
+            # Clean up temporary file if it was created for a URL
+            if (
+                file_path != str(Path(video_source).resolve())
+                and Path(file_path).exists()
+            ):
+                Path(file_path).unlink()
+
+            if not frames:
+                raise ValueError(
+                    f"Could not extract any frames from video: {video_source}"
+                )
+
+            return frames
+
+        except Exception as e:
+            self._color_log(
+                f"Error extracting frames from {video_source}: {str(e)}", Color.red
+            )
+            raise
+
+    def _create_video_content(
+        self, prompt: str, video_frames: list[dict[str, any]]
+    ) -> list[dict[str, any]]:
+        """Create uniform video format for querying LLM."""
+        content = [{"type": "text", "text": prompt}]
+        content.extend(
+            [
+                {"type": "image_url", "image_url": {"url": frame["data"]}}
+                for frame in video_frames
+            ]
+        )
+        return content
+
+    def _format_analysis_output(
+        self, result: VideoAnalysisResult, format_type: str = "markdown"
+    ) -> str:
+        """Format video analysis results for LLM consumption.
+
+        Args:
+            result: Video analysis result
+            format_type: Output format ('markdown', 'json', 'text')
+
+        Returns:
+            Formatted string suitable for LLM consumption
+        """
+        if not result.success:
+            return f"Failed to analyze video: {result.error}"
+
+        if format_type == "json":
+            return result.model_dump_json(indent=2)
+
+        elif format_type == "text":
+            output_parts = [
+                "Video Analysis Results",
+                f"Source: {result.video_source}",
+                f"Frames Analyzed: {result.frame_count}",
+                f"Duration: {result.duration_analyzed:.2f} seconds",
+                "",
+                "Analysis:",
+                result.analysis_result,
+            ]
+            return "\n".join(output_parts)
+
+        else:  # markdown (default)
+            output_parts = [
+                "# Video Analysis Results ✅",
+                "",
+                "## Video Information",
+                f"**Source:** `{result.video_source}`",
+                f"**Frames Analyzed:** {result.frame_count}",
+                f"**Duration:** {result.duration_analyzed:.2f} seconds",
+                "",
+                "## Analysis Results",
+                result.analysis_result,
+            ]
+            return "\n".join(output_parts)
+
+    def _format_summary_output(
+        self, result: VideoSummaryResult, format_type: str = "markdown"
+    ) -> str:
+        """Format video summary results for LLM consumption.
+
+        Args:
+            result: Video summary result
+            format_type: Output format ('markdown', 'json', 'text')
+
+        Returns:
+            Formatted string suitable for LLM consumption
+        """
+        if not result.success:
+            return f"Failed to summarize video: {result.error}"
+
+        if format_type == "json":
+            return result.model_dump_json(indent=2)
+
+        elif format_type == "text":
+            output_parts = [
+                "Video Summary",
+                f"Source: {result.video_source}",
+                f"Frames Analyzed: {result.frame_count}",
+                f"Duration: {result.duration_analyzed:.2f} seconds",
+                "",
+                "Summary:",
+                result.summary,
+            ]
+            return "\n".join(output_parts)
+
+        else:  # markdown (default)
+            output_parts = [
+                "# Video Summary ✅",
+                "",
+                "## Video Information",
+                f"**Source:** `{result.video_source}`",
+                f"**Frames Analyzed:** {result.frame_count}",
+                f"**Duration:** {result.duration_analyzed:.2f} seconds",
+                "",
+                "## Summary",
+                result.summary,
+            ]
+            return "\n".join(output_parts)
+
+    def _format_keyframe_output(
+        self, result: KeyframeResult, format_type: str = "markdown"
+    ) -> str:
+        """Format keyframe extraction results for LLM consumption.
+
+        Args:
+            result: Keyframe extraction result
+            format_type: Output format ('markdown', 'json', 'text')
+
+        Returns:
+            Formatted string suitable for LLM consumption
+        """
+        if not result.success:
+            return f"Failed to extract keyframes: {result.error}"
+
+        if format_type == "json":
+            return result.model_dump_json(indent=2)
+
+        elif format_type == "text":
+            output_parts = [
+                "Keyframe Extraction Results",
+                f"Target Time: {result.target_time}s",
+                f"Window Size: {result.window_size}s",
+                f"Frames Extracted: {result.frame_count}",
+                f"Output Directory: {result.output_directory}",
+                "",
+                "Frame Files:",
+            ]
+            for i, (path, timestamp) in enumerate(
+                zip(result.frame_paths, result.frame_timestamps), 1
+            ):
+                output_parts.append(f"{i}. {path} (at {timestamp:.2f}s)")
+
+            return "\n".join(output_parts)
+
+        else:  # markdown (default)
+            output_parts = [
+                "# Keyframe Extraction Results ✅",
+                "",
+                "## Extraction Parameters",
+                f"**Target Time:** {result.target_time}s",
+                f"**Window Size:** {result.window_size}s",
+                f"**Frames Extracted:** {result.frame_count}",
+                f"**Output Directory:** `{result.output_directory}`",
+                "",
+                "## Extracted Frames",
+            ]
+
+            for i, (path, timestamp) in enumerate(
+                zip(result.frame_paths, result.frame_timestamps), 1
+            ):
+                output_parts.append(f"{i}. `{path}` (at {timestamp:.2f}s)")
+
+            return "\n".join(output_parts)
+
+    def _analyze_frame_chunk(
+        self, chunk_data: tuple[int, list, str]
+    ) -> tuple[int, str]:
+        """Analyze a chunk of video frames using LLM.
+
+        Args:
+            chunk_data: Tuple containing (chunk_index, frames, question)
+
+        Returns:
+            Tuple of (chunk_index, analysis_result)
+        """
+        chunk_index, frames, question = chunk_data
+
+        try:
+            content = self._create_video_content(
+                self.video_analyze_prompt.format(task=question), frames
+            )
+            inputs = [{"role": "user", "content": content}]
+
+            response: ModelResponse = call_llm_model(
+                get_llm_model(
+                    conf=AgentConfig(
+                        llm_provider="openai",
+                        llm_model_name=os.getenv("VIDEO_LLM_MODEL_NAME"),
+                        llm_api_key=os.getenv("VIDEO_LLM_API_KEY"),
+                        llm_base_url=os.getenv("VIDEO_LLM_BASE_URL"),
+                    )
+                ),
+                inputs,
+                temperature=float(os.getenv("VIDEO_LLM_TEMPERATURE", "1.0")),
+            )
+            analysis_result = response.content
+            self._color_log(
+                f"✅ Completed analysis for chunk {chunk_index + 1}", Color.green
+            )
+
+        except Exception as e:
+            self._color_log(
+                f"❌ LLM analysis error for chunk {chunk_index + 1}: {str(e)}",
+                Color.yellow,
+            )
+            analysis_result = (
+                f"Analysis failed for video segment {chunk_index + 1}: {str(e)}"
+            )
+
+        return chunk_index, analysis_result
+
+    async def mcp_analyze_video(
+        self,
+        video_url: str = Field(description="Path or URL to the video file to analyze"),
+        question: str = Field(description="Question or task for video analysis"),
+        sample_rate: float = Field(
+            default=1.0, description="Frame sampling rate (frames per second)"
+        ),
+        start_time: float = Field(default=0.0, description="Start time in seconds"),
+        end_time: float | None = Field(
+            default=None, description="End time in seconds (None for full video)"
+        ),
+        output_format: str = Field(
+            default="markdown",
+            description="Output format: 'markdown', 'json', or 'text'",
+        ),
+        max_workers: int = Field(
+            default=4, description="Maximum number of parallel workers for analysis"
+        ),
+    ) -> ActionResponse:
+        """Analyze video content using AI with parallel processing.
+
+        This tool provides comprehensive video analysis capabilities including:
+        - Content understanding and description
+        - Object and scene detection
+        - Action and movement analysis
+        - Temporal event tracking
+        - Question-answering about video content
+        - Parallel processing for faster analysis
+
+        Args:
+            video_url: Path or URL to the video file
+            question: Specific question or analysis task
+            sample_rate: Frame sampling rate for analysis
+            start_time: Start time of the video segment in seconds
+            end_time: End time of the video segment in seconds
+            output_format: Format for the response output
+            max_workers: Maximum number of parallel workers
+
+        Returns:
+            ActionResponse with video analysis results and metadata
+        """
+        start_exec_time = time.time()
+
+        try:
+            # Validate video file
+            video_path = self._validate_file_path(video_url)
+
+            self._color_log(f"🎬 Analyzing video: {video_url}", Color.cyan)
+            self._color_log(f"📋 Question: {question}", Color.blue)
+
+            # Extract video frames
+            video_frames = self._get_video_frames(
+                str(video_path), sample_rate, start_time, end_time
+            )
+            self._color_log(f"📸 Extracted {len(video_frames)} frames", Color.blue)
+
+            # Process frames in chunks of 64 frames for parallel analysis
+            chunk_size = 128
+            chunks = []
+
+            # Create chunks of `chunk_size` continuous frames
+            for i in range(0, len(video_frames), chunk_size):
+                chunk_frames = video_frames[i : i + chunk_size]
+                chunks.append((i // chunk_size, chunk_frames, question))
+
+            self._color_log(
+                f"🔄 Processing {len(chunks)} chunks with {max_workers} parallel workers",
+                Color.blue,
+            )
+
+            # Process chunks in parallel
+            all_results = [None] * len(chunks)  # Pre-allocate to maintain order
+
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                # Submit all chunk analysis tasks
+                future_to_chunk = {
+                    executor.submit(self._analyze_frame_chunk, chunk_data): chunk_data[
+                        0
+                    ]
+                    for chunk_data in chunks
+                }
+
+                # Collect results as they complete
+                for future in as_completed(future_to_chunk):
+                    try:
+                        chunk_index, result = future.result()
+                        all_results[chunk_index] = (
+                            f"Result of video part {chunk_index + 1}: {result}"
+                        )
+                    except Exception as e:
+                        chunk_index = future_to_chunk[future]
+                        self._color_log(
+                            f"❌ Error processing chunk {chunk_index + 1}: {str(e)}",
+                            Color.red,
+                        )
+                        all_results[chunk_index] = (
+                            f"Result of video part {chunk_index + 1}: Analysis failed - {str(e)}"
+                        )
+
+            # Filter out None results and join
+            analysis_result = "\n".join(
+                [result for result in all_results if result is not None]
+            )
+            duration_analyzed = (
+                end_time - start_time if end_time else len(video_frames) / sample_rate
+            )
+
+            # Create result
+            result = VideoAnalysisResult(
+                video_source=video_url,
+                analysis_result=analysis_result,
+                frame_count=len(video_frames),
+                duration_analyzed=duration_analyzed,
+                success=True,
+                error=None,
+            )
+
+            # Format output for LLM
+            message = self._format_analysis_output(result, output_format)
+            execution_time = time.time() - start_exec_time
+
+            # Create metadata
+            metadata = {
+                "video_source": video_url,
+                "frame_count": len(video_frames),
+                "chunks_processed": len(chunks),
+                "chunk_size": chunk_size,
+                "parallel_workers": max_workers,
+                "duration_analyzed": duration_analyzed,
+                "sample_rate": sample_rate,
+                "start_time": start_time,
+                "end_time": end_time,
+                "execution_time": execution_time,
+                "output_format": output_format,
+                "success": True,
+            }
+
+            self._color_log(
+                f"✅ Video analysis completed in {execution_time:.2f}s "
+                f"({len(chunks)} chunks, {len(video_frames)} frames)",
+                Color.green,
+            )
+
+            return ActionResponse(success=True, message=message, metadata=metadata)
+
+        except Exception as e:
+            execution_time = time.time() - start_exec_time
+            error_msg = f"Video analysis failed: {str(e)}"
+            self._color_log(f"❌ {error_msg}", Color.red)
+            self.logger.error(f"{error_msg}: {traceback.format_exc()}")
+
+            return ActionResponse(
+                success=False,
+                message=error_msg,
+                metadata={
+                    "video_source": video_url,
+                    "execution_time": execution_time,
+                    "error": str(e),
+                    "success": False,
+                },
+            )
+
+    async def mcp_summarize_video(
+        self,
+        video_url: str = Field(
+            description="The input video filepath or URL to summarize."
+        ),
+        sample_rate: int = Field(
+            default=1, description="Sample n frames per second (default: 1)."
+        ),
+        start_time: float = Field(
+            default=0,
+            description="Start time of the video segment in seconds (default: 0).",
+        ),
+        end_time: float | None = Field(
+            default=None,
+            description="End time of the video segment in seconds (default: None).",
+        ),
+        output_format: str = Field(
+            default="markdown",
+            description="Output format: 'markdown', 'json', or 'text' (default: markdown).",
+        ),
+    ) -> ActionResponse:
+        """Summarize the main content of a video using AI analysis.
+
+        This tool provides AI-powered video summarization with:
+        - Key point extraction
+        - Main topic identification
+        - Important visual element recognition
+        - LLM-optimized result formatting
+
+        Args:
+            video_url: The input video filepath or URL to summarize
+            sample_rate: Sample n frames per second
+            start_time: Start time of the video segment in seconds
+            end_time: End time of the video segment in seconds
+            output_format: Format for the response output
+
+        Returns:
+            ActionResponse with video summary results and metadata
+        """
+        start_exec_time = time.time()
+
+        try:
+            # Validate video file
+            video_path = self._validate_file_path(video_url)
+
+            self._color_log(f"🎬 Summarizing video: {video_url}", Color.cyan)
+
+            # Extract video frames
+            video_frames = self._get_video_frames(
+                str(video_path), sample_rate, start_time, end_time
+            )
+            self._color_log(f"📸 Extracted {len(video_frames)} frames", Color.blue)
+
+            # Process frames in larger chunks for summarization
+            interval = 490
+            frame_nums = 500
+            all_results = []
+
+            for i in range(0, len(video_frames), interval):
+                cur_frames = video_frames[i : i + frame_nums]
+                content = self._create_video_content(
+                    self.video_summarize_prompt, cur_frames
+                )
+                inputs = [{"role": "user", "content": content}]
+
+                try:
+                    response: ModelResponse = call_llm_model(
+                        get_llm_model(
+                            conf=AgentConfig(
+                                llm_provider="openai",
+                                llm_model_name=os.getenv(
+                                    "VIDEO_LLM_MODEL_NAME", "gpt-4o"
+                                ),
+                                llm_api_key=os.getenv("VIDEO_LLM_API_KEY"),
+                                llm_base_url=os.getenv("VIDEO_LLM_BASE_URL"),
+                            )
+                        ),
+                        inputs,
+                        temperature=float(os.getenv("VIDEO_LLM_TEMPERATURE", "1.0")),
+                    )
+                    cur_summary = response.content
+                except Exception as e:
+                    self._color_log(
+                        f"LLM summary error for chunk {i // interval + 1}: {str(e)}",
+                        Color.yellow,
+                    )
+                    cur_summary = (
+                        f"Summary failed for video segment {i // interval + 1}"
+                    )
+
+                all_results.append(
+                    f"Summary of video part {i // interval + 1}: {cur_summary}"
+                )
+
+                if i + frame_nums >= len(video_frames):
+                    break
+
+            summary_result = "\n".join(all_results)
+            duration_analyzed = (
+                end_time - start_time if end_time else len(video_frames) / sample_rate
+            )
+
+            # Create result
+            result = VideoSummaryResult(
+                video_source=video_url,
+                summary=summary_result,
+                frame_count=len(video_frames),
+                duration_analyzed=duration_analyzed,
+                success=True,
+                error=None,
+            )
+
+            # Format output for LLM
+            message = self._format_summary_output(result, output_format)
+            execution_time = time.time() - start_exec_time
+
+            # Create metadata
+            metadata = VideoMetadata(
+                operation="summarize",
+                video_source=video_url,
+                sample_rate=sample_rate,
+                start_time=start_time,
+                end_time=end_time,
+                frame_count=len(video_frames),
+                execution_time=execution_time,
+            ).model_dump()
+
+            self._color_log(
+                "✅ Video summarization completed successfully", Color.green
+            )
+            return ActionResponse(success=True, message=message, metadata=metadata)
+
+        except Exception as e:
+            error_msg = str(e)
+            self._color_log(
+                f"❌ Video summarization error: {traceback.format_exc()}", Color.red
+            )
+
+            # Format error for LLM
+            message = f"Failed to summarize video: {error_msg}"
+            execution_time = time.time() - start_exec_time
+
+            # Create metadata
+            metadata = VideoMetadata(
+                operation="summarize",
+                video_source=video_url,
+                error_type="summarization_failure",
+                execution_time=execution_time,
+            ).model_dump()
+
+            return ActionResponse(success=False, message=message, metadata=metadata)
+
+    async def mcp_extract_keyframes(
+        self,
+        video_path: str = Field(description="The input video filepath or URL."),
+        target_time: int = Field(
+            description="The specific time point for extraction (in seconds), centered within the window_size."
+        ),
+        window_size: int = Field(
+            default=5,
+            description="The window size for extraction (in seconds, default: 5).",
+        ),
+        output_dir: str = Field(
+            default=None,
+            description="Directory where extracted frames will be saved (default: workspace/keyframes).",
+        ),
+        output_format: str = Field(
+            default="markdown",
+            description="Output format: 'markdown', 'json', or 'text' (default: markdown).",
+        ),
+    ) -> ActionResponse:
+        """Extract key frames around a target time with scene detection.
+
+        This tool provides keyframe extraction with:
+        - Scene detection for significant frame changes
+        - Configurable time windows
+        - Automatic output directory management
+        - LLM-optimized result formatting
+
+        Args:
+            video_path: The input video filepath or URL
+            target_time: Specific time point (in seconds) to extract frames around
+            window_size: Time window (in seconds) centered on target_time
+            cleanup: Whether to delete the original video file after processing
+            output_dir: Directory where extracted frames will be saved
+            output_format: Format for the response output
+
+        Returns:
+            ActionResponse with keyframe extraction results and metadata
+        """
+        start_exec_time = time.time()
+
+        try:
+            # Validate video file
+            validated_path = self._validate_file_path(video_path)
+
+            # Set default output directory
+            output_dir = (
+                str(self.workspace / "keyframes") if output_dir is None else output_dir
+            )
+
+            output_path = Path(output_dir)
+            output_path.mkdir(parents=True, exist_ok=True)
+
+            self._color_log(f"🎬 Extracting keyframes from: {video_path}", Color.cyan)
+            self._color_log(
+                f"🎯 Target time: {target_time}s, Window: {window_size}s", Color.blue
+            )
+
+            # Extract keyframes with scene detection
+            frames, frame_times = self._extract_keyframes_with_scene_detection(
+                str(validated_path), target_time, window_size
+            )
+
+            # Save frames to disk
+            frame_paths, frame_timestamps = self._save_keyframes(
+                frames, frame_times, str(output_path)
+            )
+
+            # Cleanup if requested
+            # if cleanup and validated_path.exists():
+            #     validated_path.unlink()
+            #     self._color_log("🗑️ Cleaned up original video file", Color.yellow)
+
+            # Create result
+            result = KeyframeResult(
+                frame_paths=frame_paths,
+                frame_timestamps=frame_timestamps,
+                output_directory=str(output_path),
+                frame_count=len(frame_paths),
+                target_time=float(target_time),
+                window_size=float(window_size),
+                success=True,
+                error=None,
+            )
+
+            # Format output for LLM
+            message = self._format_keyframe_output(result, output_format)
+            execution_time = time.time() - start_exec_time
+
+            # Create metadata
+            metadata = VideoMetadata(
+                operation="extract_keyframes",
+                video_source=video_path,
+                target_time=float(target_time),
+                window_size=float(window_size),
+                output_directory=str(output_path),
+                frame_count=len(frame_paths),
+                execution_time=execution_time,
+            ).model_dump()
+
+            self._color_log(
+                f"✅ Extracted {len(frame_paths)} keyframes successfully", Color.green
+            )
+            return ActionResponse(success=True, message=message, metadata=metadata)
+
+        except Exception as e:
+            error_msg = str(e)
+            self._color_log(
+                f"❌ Keyframe extraction error: {traceback.format_exc()}", Color.red
+            )
+
+            # Format error for LLM
+            message = f"Failed to extract keyframes: {error_msg}"
+            execution_time = time.time() - start_exec_time
+
+            # Create metadata
+            metadata = VideoMetadata(
+                operation="extract_keyframes",
+                video_source=video_path,
+                target_time=float(target_time) if target_time else None,
+                window_size=float(window_size) if window_size else None,
+                error_type="keyframe_extraction_failure",
+                execution_time=execution_time,
+            ).model_dump()
+
+            return ActionResponse(success=False, message=message, metadata=metadata)
+
+    def _extract_keyframes_with_scene_detection(
+        self, video_path: str, target_time: int, window_size: int
+    ) -> tuple[list[any], list[float]]:
+        """Extract key frames around the target time with scene detection.
+
+        Args:
+            video_path: Path to the video file
+            target_time: Target time in seconds
+            window_size: Window size in seconds
+
+        Returns:
+            Tuple of (frames, frame_times)
+        """
+        cap = cv2.VideoCapture(video_path)  # pylint: disable=E1101
+        fps = cap.get(cv2.CAP_PROP_FPS)  # pylint: disable=E1101
+
+        # Calculate frame numbers for the time window
+        start_frame = int((target_time - window_size / 2) * fps)
+        end_frame = int((target_time + window_size / 2) * fps)
+        total_frames_in_window = end_frame - start_frame
+
+        max_frames = 384  # Maximum allowed frames to prevent memory issues
+
+        # Calculate sampling interval for even distribution
+        if total_frames_in_window <= max_frames:
+            # If total frames is within limit, use scene detection normally
+            frame_interval = 1
+            use_scene_detection = True
+        else:
+            # If exceeds limit, sample evenly across the window
+            frame_interval = total_frames_in_window // max_frames
+            use_scene_detection = False  # Skip scene detection for even sampling
+
+        frames = []
+        frame_times = []
+
+        # Set video position to start_frame
+        cap.set(cv2.CAP_PROP_POS_FRAMES, max(0, start_frame))  # pylint: disable=E1101
+
+        prev_frame = None
+        frame_count = 0
+
+        while cap.isOpened() and len(frames) < max_frames:
+            frame_pos = cap.get(cv2.CAP_PROP_POS_FRAMES)  # pylint: disable=E1101
+            if frame_pos >= end_frame:
+                break
+
+            ret, frame = cap.read()
+            if not ret:
+                break
+
+            if use_scene_detection:
+                # Use original scene detection logic
+                gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)  # pylint: disable=E1101
+
+                # If this is the first frame, save it
+                if prev_frame is None:
+                    frames.append(frame)
+                    frame_times.append(frame_pos / fps)
+                else:
+                    # Calculate difference between current and previous frame
+                    diff = cv2.absdiff(gray, prev_frame)  # pylint: disable=E1101
+                    mean_diff = np.mean(diff)
+
+                    # If significant change detected, save frame
+                    if mean_diff > 20:  # Threshold for scene change
+                        frames.append(frame)
+                        frame_times.append(frame_pos / fps)
+
+                prev_frame = gray
+            else:
+                # Use even sampling for large windows
+                if frame_count % frame_interval == 0:
+                    frames.append(frame)
+                    frame_times.append(frame_pos / fps)
+
+                frame_count += 1
+
+        cap.release()
+        return frames, frame_times
+
+    def _save_keyframes(
+        self, frames: list[any], frame_times: list[float], output_dir: str
+    ) -> tuple[list[str], list[float]]:
+        """Save extracted frames to disk.
+
+        Args:
+            frames: List of frame objects
+            frame_times: List of frame timestamps
+            output_dir: Output directory path
+
+        Returns:
+            Tuple of (saved_paths, saved_timestamps)
+        """
+        output_path = Path(output_dir)
+        output_path.mkdir(parents=True, exist_ok=True)
+
+        saved_paths = []
+        saved_timestamps = []
+
+        for _, (frame, timestamp) in enumerate(zip(frames, frame_times)):
+            filename = output_path / f"frame_{timestamp:.2f}s.jpg"
+            cv2.imwrite(str(filename), frame)  # pylint: disable=E1101
+            saved_paths.append(str(filename))
+            saved_timestamps.append(timestamp)
+
+        return saved_paths, saved_timestamps
+
+
+# Default arguments for testing
+if __name__ == "__main__":
+    load_dotenv()
+
+    arguments = ActionArguments(
+        name="video",
+        transport="stdio",
+        workspace=os.getenv("AWORLD_WORKSPACE", "~"),
+    )
+
+    try:
+        service = VideoCollection(arguments)
+        service.run()
+    except Exception as e:
+        print(f"An error occurred: {str(e)}")