ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter9/gaia-experience/AWorld/aworld/trace/__init__.py

@@ -0,0 +1,142 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+from aworld.trace.context_manager import TraceManager
+from aworld.trace.constants import (
+    SPAN_NAME_PREFIX_EVENT_AGENT,
+    SPAN_NAME_PREFIX_EVENT_TOOL,
+    SPAN_NAME_PREFIX_EVENT_TASK,
+    SPAN_NAME_PREFIX_EVENT_OUTPUT,
+    SPAN_NAME_PREFIX_EVENT_OTHER,
+    SPAN_NAME_PREFIX_AGENT,
+    SPAN_NAME_PREFIX_TOOL,
+    ATTRIBUTES_MESSAGE_RUN_TYPE_KEY,
+    SPAN_NAME_PREFIX_TASK,
+    RunType
+)
+from aworld.trace.instrumentation.agent import get_agent_span_attributes
+from aworld.trace.instrumentation.tool import get_tool_name, get_tool_span_attributes
+from aworld.trace.instrumentation import semconv
+from aworld.trace.instrumentation.uni_llmmodel.model_response_parse import covert_to_jsonstr
+from aworld.trace.config import configure, ObservabilityConfig
+from typing import Callable, Any
+
+
+def get_span_name_from_message(message: 'aworld.core.event.base.Message') -> tuple[str, RunType]:
+    from aworld.core.event.base import Constants
+    span_name = (message.receiver or message.id)
+    if message.category == Constants.AGENT:
+        return (SPAN_NAME_PREFIX_EVENT_AGENT + span_name, RunType.AGNET)
+    if message.category == Constants.TOOL:
+        action = message.payload
+        if isinstance(action, (list, tuple)):
+            action = action[0]
+        if action:
+            tool_name, run_type = get_tool_name(action.tool_name, action)
+            return (SPAN_NAME_PREFIX_EVENT_TOOL + tool_name, run_type)
+        return (SPAN_NAME_PREFIX_EVENT_TOOL + span_name, RunType.TOOL)
+    if message.category == Constants.TASK:
+        if message.topic:
+            return (SPAN_NAME_PREFIX_EVENT_TASK + message.topic + "_" + span_name, RunType.OTHER)
+        else:
+            return (SPAN_NAME_PREFIX_EVENT_TASK + span_name, RunType.OTHER)
+    if message.category == Constants.OUTPUT:
+        return (SPAN_NAME_PREFIX_EVENT_OUTPUT + span_name, RunType.OTHER)
+    return (SPAN_NAME_PREFIX_EVENT_OTHER + span_name, RunType.OTHER)
+
+
+def message_span(message: 'aworld.core.event.base.Message' = None, attributes: dict = None):
+    if message:
+        span_name, run_type = get_span_name_from_message(message)
+        message_span_attribute = {
+            "event.payload": str(message.payload),
+            "event.topic": message.topic or "",
+            "event.receiver": message.receiver or "",
+            "event.sender": message.sender or "",
+            "event.category": message.category,
+            "event.id": message.id,
+            semconv.SESSION_ID: message.session_id
+        }
+        message_span_attribute.update(attributes or {})
+        return GLOBAL_TRACE_MANAGER.span(
+            span_name=span_name,
+            attributes=message_span_attribute,
+            run_type=run_type
+        )
+    else:
+        raise ValueError("message_span message is None")
+
+
+def handler_span(message: 'aworld.core.event.base.Message' = None, handler: Callable[..., Any] = None, attributes: dict = None):
+    from aworld.core.event.base import Constants
+    attributes = attributes or {}
+    span_name = handler.__name__
+    if message:
+        run_type = RunType.OTHER
+        if message.category == Constants.AGENT:
+            span_name = SPAN_NAME_PREFIX_AGENT + span_name
+            run_type = RunType.AGNET
+            attributes.update(get_agent_span_attributes(handler.__self__, message))
+        if message.category == Constants.TOOL:
+            span_name = SPAN_NAME_PREFIX_TOOL + span_name
+            run_type = RunType.TOOL
+            attributes.update(get_tool_span_attributes(handler.__self__, message))
+            if attributes.get(ATTRIBUTES_MESSAGE_RUN_TYPE_KEY):
+                run_type = RunType[attributes.get(ATTRIBUTES_MESSAGE_RUN_TYPE_KEY)]
+        return GLOBAL_TRACE_MANAGER.span(
+            span_name=span_name,
+            attributes=attributes,
+            run_type=run_type
+        )
+    else:
+        return GLOBAL_TRACE_MANAGER.span(
+            span_name=span_name,
+            attributes=attributes
+        )
+
+
+def task_span(session_id: str, task: 'aworld.core.task.Task' = None, attributes: dict = None):
+    attributes = attributes or {}
+    if task:
+        message_span_attribute = {
+            semconv.SESSION_ID: task.session_id,
+            semconv.TASK_ID: task.id,
+            semconv.TASK_INPUT: task.input,
+            semconv.TASK_IS_SUB_TASK: task.is_sub_task,
+            semconv.TASK_GROUP_ID: task.group_id,
+            semconv.TASK: covert_to_jsonstr(task)
+        }
+        message_span_attribute.update(attributes)
+        return GLOBAL_TRACE_MANAGER.span(
+            span_name=SPAN_NAME_PREFIX_TASK + task.id,
+            attributes=message_span_attribute,
+            run_type=RunType.TASK
+        )
+    else:
+        message_span_attribute = {
+            semconv.SESSION_ID: task.session_id
+        }
+        return GLOBAL_TRACE_MANAGER.span(
+            span_name=SPAN_NAME_PREFIX_TASK + session_id,
+            attributes=attributes,
+            run_type=RunType.TASK
+        )
+
+
+GLOBAL_TRACE_MANAGER: TraceManager = TraceManager()
+span = GLOBAL_TRACE_MANAGER.span
+func_span = GLOBAL_TRACE_MANAGER.func_span
+auto_tracing = GLOBAL_TRACE_MANAGER.auto_tracing
+get_current_span = GLOBAL_TRACE_MANAGER.get_current_span
+new_manager = GLOBAL_TRACE_MANAGER.get_current_span
+
+__all__ = [
+    "span",
+    "func_span",
+    "message_span",
+    "auto_tracing",
+    "get_current_span",
+    "new_manager",
+    "RunType",
+    "configure",
+    "ObservabilityConfig"
+]

chapter9/gaia-experience/AWorld/aworld/trace/auto_trace.py

@@ -0,0 +1,192 @@
+import ast
+import re
+import sys
+import warnings
+from importlib.abc import Loader, MetaPathFinder
+from importlib.machinery import ModuleSpec
+from importlib.util import spec_from_loader
+from types import ModuleType
+from typing import TYPE_CHECKING, Sequence, Union, Callable, Iterator, TypeVar, Any, cast
+
+from aworld.trace.base import log_trace_error
+from .rewrite_ast import compile_source
+
+if TYPE_CHECKING:
+    from .context_manager import TraceManager
+
+
+class AutoTraceModule:
+    """A class that represents a module being imported that should maybe be traced automatically."""
+
+    def __init__(self, module_name: str) -> None:
+        self._module_name = module_name
+        """Fully qualified absolute name of the module being imported."""
+
+    def need_auto_trace(self, prefix: Union[str, Sequence[str]]) -> bool:
+        """
+        Check if the module name starts with the given prefix.
+        """
+        if isinstance(prefix, str):
+            prefix = (prefix,)
+        pattern = '|'.join([get_module_pattern(p) for p in prefix])
+        return bool(re.match(pattern, self._module_name))
+
+
+class TraceImportFinder(MetaPathFinder):
+    """A class that implements the `find_spec` method of the `MetaPathFinder` protocol."""
+
+    def __init__(self, trace_manager: "TraceManager", module_funcs: Callable[[AutoTraceModule], bool],
+                 min_duration_ns: int) -> None:
+        self._trace_manager = trace_manager
+        self._modules_filter = module_funcs
+        self._min_duration_ns = min_duration_ns
+
+    def _find_plain_specs(
+            self, fullname: str, path: Sequence[str] = None, target: ModuleType = None
+    ) -> Iterator[ModuleSpec]:
+        """Yield module specs returned by other finders on `sys.meta_path`."""
+        for finder in sys.meta_path:
+            # Skip this finder or any like it to avoid infinite recursion.
+            if isinstance(finder, TraceImportFinder):
+                continue
+
+            try:
+                plain_spec = finder.find_spec(fullname, path, target)
+            except Exception:  # pragma: no cover
+                continue
+
+            if plain_spec:
+                yield plain_spec
+
+    def find_spec(self, fullname: str, path: Sequence[str], target=None) -> None:
+        """Find the spec for the given module name."""
+
+        for plain_spec in self._find_plain_specs(fullname, path, target):
+            # Get module specs returned by other finders on `sys.meta_path`
+            get_source = getattr(plain_spec.loader, 'get_source', None)
+            if not callable(get_source):
+                continue
+            try:
+                source = cast(str, get_source(fullname))
+            except Exception:
+                continue
+
+            if not source:
+                continue
+
+            filename = plain_spec.origin
+            if not filename:
+                try:
+                    filename = cast('str | None', plain_spec.loader.get_filename(fullname))
+                except Exception:
+                    pass
+            filename = filename or f'<{fullname}>'
+
+            if not self._modules_filter(AutoTraceModule(fullname)):
+                return None
+
+            try:
+                tree = ast.parse(source)
+            except Exception:
+                # Invalid source code. Try another one.
+                continue
+
+            try:
+                execute = compile_source(tree, filename, fullname, self._trace_manager, self._min_duration_ns)
+            except Exception:  # pragma: no cover
+                log_trace_error()
+                return None
+
+            loader = AutoTraceLoader(plain_spec, execute)
+            return spec_from_loader(fullname, loader)
+
+
+class AutoTraceLoader(Loader):
+    """
+    A class that implements the `exec_module` method of the `Loader` protocol.
+    """
+
+    def __init__(self, plain_spec: ModuleSpec, execute: Callable[[dict[str, Any]], None]) -> None:
+        self._plain_spec = plain_spec
+        self._execute = execute
+
+    def exec_module(self, module: ModuleType):
+        """Execute a modified AST of the module's source code in the module's namespace.
+        """
+        self._execute(module.__dict__)
+
+    def create_module(self, spec: ModuleSpec):
+        return None
+
+    def get_code(self, _name: str):
+        """`python -m` uses the `runpy` module which calls this method instead of going through the normal protocol.
+        So return some code which can be executed with the module namespace.
+        Here `__loader__` will be this object, i.e. `self`.
+        source = '__loader__.execute(globals())'
+        return compile(source, '<string>', 'exec', dont_inherit=True)
+        """
+
+    def __getattr__(self, item: str):
+        """Forward some methods to the plain spec's loader (likely a `SourceFileLoader`) if they exist."""
+        if item in {'get_filename', 'is_package'}:
+            return getattr(self.plain_spec.loader, item)
+        raise AttributeError(item)
+
+
+def convert_to_modules_func(modules: Sequence[str]) -> Callable[[AutoTraceModule], bool]:
+    """Convert a sequence of module names to a function that checks if a module name starts with any of the given module names.
+    """
+    return lambda module: module.need_auto_trace(modules)
+
+
+def get_module_pattern(module: str):
+    """
+    Get the regex pattern for the given module name.
+    """
+
+    if not re.match(r'[\w.]+$', module, re.UNICODE):
+        return module
+    module = re.escape(module)
+    return rf'{module}($|\.)'
+
+
+def install_auto_tracing(trace_manager: "TraceManager",
+                         modules: Union[Sequence[str],
+                         Callable[[AutoTraceModule], bool]],
+                         min_duration_seconds: float
+                         ) -> None:
+    """
+    Automatically trace the execution of a function.
+    """
+    if isinstance(modules, Sequence):
+        module_funcs = convert_to_modules_func(modules)
+    else:
+        module_funcs = modules
+
+    if not callable(module_funcs):
+        raise TypeError('modules must be a list of strings or a callable')
+
+    for module in list(sys.modules.values()):
+        try:
+            auto_trace_module = AutoTraceModule(module.__name__)
+        except Exception:
+            continue
+
+        if module_funcs(auto_trace_module):
+            warnings.warn(f'The module {module.__name__!r} matches modules to trace, but it has already been imported. '
+                          f'Call `auto_tracing` earlier',
+                          stacklevel=2,
+                          )
+
+    min_duration_ns = int(min_duration_seconds * 1_000_000_000)
+    trace_manager = trace_manager.new_manager('auto_tracing')
+    finder = TraceImportFinder(trace_manager, module_funcs, min_duration_ns)
+    sys.meta_path.insert(0, finder)
+
+
+T = TypeVar('T')
+
+
+def not_auto_trace(x: T) -> T:
+    """Decorator to prevent a function/class from being traced by `auto_tracing`"""
+    return x

chapter9/gaia-experience/AWorld/aworld/trace/baggage/__init__.py

@@ -0,0 +1,51 @@
+from uuid import uuid4
+from contextvars import ContextVar
+from types import MappingProxyType
+
+_BAGGAGE_KEY = "aworld.baggage." + str(uuid4())
+
+_BAGGAGE_CONTEXT = ContextVar(_BAGGAGE_KEY, default=None)
+
+
+class BaggageContext:
+    """
+    Baggage context.
+    """
+
+    @staticmethod
+    def get_baggage() -> dict:
+        """
+        Get the baggage. This is a read-only view of the baggage.
+        Returns:
+            The baggage.
+        """
+        baggage = _BAGGAGE_CONTEXT.get()
+        if isinstance(baggage, dict):
+            return MappingProxyType(baggage)
+        return {}
+
+    @staticmethod
+    def get_baggage_value(key: str):
+        """
+        Get the value for a key from baggage.
+        Args:
+            key: The key of the value to retrieve.
+        Returns:
+            The baggage value.
+        """
+        baggage = BaggageContext.get_baggage()
+        if key:
+            return baggage.get(key)
+        return None
+
+    @staticmethod
+    def set_baggage(key: str, value: object):
+        """
+        Set the value for a key in baggage.
+        Args:
+            key: The key of the value to set.
+            value: The value to set.
+        """
+        baggage = BaggageContext.get_baggage().copy()
+        baggage[key] = value
+        _BAGGAGE_CONTEXT.set(baggage)

chapter9/gaia-experience/AWorld/aworld/trace/baggage/sofa_tracer.py

@@ -0,0 +1,128 @@
+from aworld.trace.base import Propagator, Carrier, TraceContext
+from aworld.trace.baggage import BaggageContext
+from aworld.logs.util import logger
+from aworld.trace.base import AttributeValueType
+
+
+class SofaTracerBaggagePropagator(Propagator):
+    """
+    Sofa tracer baggage propagator.
+    """
+
+    _TRACE_ID_HEDER_NAMES = ["SOFA-TraceId", "sofaTraceId"]
+    _SPAN_ID_HEDER_NAMES = ["SOFA-RpcId", "sofaRpcId"]
+    _PEN_ATTRS_HEDER_NAME = "sofaPenAttrs"
+    _SYS_PEN_ATTRS_HEDER_NAME = "sysPenAttrs"
+
+    _TRACE_ID_BAGGAGE_KEY = "attributes.sofa.traceid"
+    _SPAN_ID_BAGGAGE_KEY = "attributes.sofa.rpcid"
+    _PEN_ATTRS_BAGGAGE_KEY = "attributes.sofa.penattrs"
+    _SYS_PEN_ATTRS_BAGGAGE_KEY = "attributes.sofa.syspenattrs"
+
+    def extract(self, carrier: Carrier):
+        """
+        Extract trace context from carrier.
+        Args:
+            carrier: The carrier to extract trace context from.
+        Returns:
+            A dict of trace context.
+        """
+        trace_id = None
+        span_id = None
+        for name in self._TRACE_ID_HEDER_NAMES:
+            trace_id = self._get_value(carrier, name)
+            if trace_id:
+                break
+        for name in self._SPAN_ID_HEDER_NAMES:
+            span_id = self._get_value(carrier, name)
+            if span_id:
+                break
+        pen_attrs = self._get_value(carrier, self._PEN_ATTRS_HEDER_NAME)
+        sys_pen_attrs = self._get_value(
+            carrier, self._SYS_PEN_ATTRS_HEDER_NAME)
+
+        logger.info(
+            f"extract trace_id: {trace_id}, span_id: {span_id}, pen_attrs: {pen_attrs}, sys_pen_attrs: {sys_pen_attrs}")
+        if trace_id and span_id:
+            BaggageContext.set_baggage(self._TRACE_ID_BAGGAGE_KEY, trace_id)
+            span_id = span_id + ".1"
+            BaggageContext.set_baggage(self._SPAN_ID_BAGGAGE_KEY, span_id)
+            if pen_attrs:
+                BaggageContext.set_baggage(
+                    self._PEN_ATTRS_BAGGAGE_KEY, pen_attrs)
+            if sys_pen_attrs:
+                BaggageContext.set_baggage(
+                    self._SYS_PEN_ATTRS_BAGGAGE_KEY, sys_pen_attrs)
+
+    def inject(self, trace_context: TraceContext, carrier: Carrier):
+        """
+        Inject trace context to carrier.
+        Args:
+            trace_context: The trace context to inject.
+            carrier: The carrier to inject trace context to.
+        """
+        baggage = BaggageContext.get_baggage()
+
+        if baggage:
+            trace_id = baggage.get(self._TRACE_ID_BAGGAGE_KEY)
+            span_id = baggage.get(self._SPAN_ID_BAGGAGE_KEY)
+            if trace_id and span_id:
+                carrier.set(self._TRACE_ID_HEDER_NAMES[0], trace_id)
+                carrier.set(self._SPAN_ID_HEDER_NAMES[0], span_id)
+
+            pen_attrs_dict = {}
+            for key, value in baggage.items():
+                if key == self._TRACE_ID_BAGGAGE_KEY or key == self._SPAN_ID_BAGGAGE_KEY:
+                    continue
+                if key == self._PEN_ATTRS_BAGGAGE_KEY and value:
+                    pen_attrs_dict.update(dict(item.split("=")
+                                          for item in value.split("&")))
+                    continue
+                if key == self._SYS_PEN_ATTRS_BAGGAGE_KEY and value:
+                    carrier.set(self._SYS_PEN_ATTRS_HEDER_NAME, value)
+                    continue
+
+                # other baggage items will be injected to sofaPenAttrs
+                pen_attrs_dict.update({key: value})
+
+            if pen_attrs_dict:
+                pen_attrs = "&".join(f"{key}={value}"
+                                     for key, value in pen_attrs_dict.items())
+                carrier.set(self._PEN_ATTRS_HEDER_NAME, pen_attrs)
+
+
+class SofaSpanHelper:
+    """
+    Sofa span helper.
+    """
+
+    @staticmethod
+    def set_sofa_context_to_attr(span_attributes: dict[str, AttributeValueType]):
+        """
+        Set sofa context to span attributes.
+        Args:
+            span_attributes: The span attributes to set sofa context to.
+        """
+        baggage = BaggageContext.get_baggage()
+        if baggage:
+            trace_id = baggage.get(
+                SofaTracerBaggagePropagator._TRACE_ID_BAGGAGE_KEY)
+            span_id = baggage.get(
+                SofaTracerBaggagePropagator._SPAN_ID_BAGGAGE_KEY)
+            if trace_id and span_id:
+                span_attributes.update({
+                    SofaTracerBaggagePropagator._TRACE_ID_BAGGAGE_KEY: trace_id,
+                    SofaTracerBaggagePropagator._SPAN_ID_BAGGAGE_KEY: span_id
+                })
+            pen_attrs = baggage.get(
+                SofaTracerBaggagePropagator._PEN_ATTRS_BAGGAGE_KEY)
+            if pen_attrs:
+                span_attributes.update({
+                    SofaTracerBaggagePropagator._PEN_ATTRS_BAGGAGE_KEY: pen_attrs
+                })
+            sys_pen_attrs = baggage.get(
+                SofaTracerBaggagePropagator._SYS_PEN_ATTRS_BAGGAGE_KEY)
+            if sys_pen_attrs:
+                span_attributes.update({
+                    SofaTracerBaggagePropagator._SYS_PEN_ATTRS_BAGGAGE_KEY: sys_pen_attrs
+                })

chapter9/gaia-experience/AWorld/aworld/trace/baggage/w3c.py

@@ -0,0 +1,65 @@
+import re
+from typing import List
+from aworld.trace.base import Propagator, Carrier, TraceContext
+from aworld.trace.baggage import BaggageContext
+from aworld.logs.util import logger
+from urllib.parse import quote_plus, unquote_plus
+
+
+class W3CBaggagePropagator(Propagator):
+    """
+    W3C baggage propagator.
+    """
+
+    _MAX_HEADER_LENGTH = 8192
+    _MAX_PAIR_LENGTH = 4096
+    _MAX_PAIRS = 180
+    _BAGGAGE_HEADER_NAME = "baggage"
+    _DELIMITER_PATTERN = re.compile(r"[ \t]*,[ \t]*")
+
+    def extract(self, carrier: Carrier):
+        """
+        Extract the trace context from the carrier.
+        Args:
+            carrier: The carrier to extract the trace context from.
+        """
+        baggage_header = self._get_value(carrier, self._BAGGAGE_HEADER_NAME)
+        if not baggage_header:
+            return None
+
+        if len(baggage_header) > self._MAX_HEADER_LENGTH:
+            logger.warning(
+                f"baggage header length exceeds {self._MAX_HEADER_LENGTH}")
+            return None
+
+        baggage_entries: List[str] = re.split(
+            self._DELIMITER_PATTERN, baggage_header)
+        if len(baggage_entries) > self._MAX_PAIRS:
+            logger.warning(f"baggage entries exceeds {self._MAX_PAIRS}")
+
+        for entry in baggage_entries:
+            if len(entry) > self._MAX_PAIR_LENGTH:
+                logger.warning(
+                    f"baggage entry length exceeds {self._MAX_PAIR_LENGTH}")
+                continue
+            try:
+                key, value = entry.split("=", 1)
+                key = unquote_plus(key).strip()
+                value = unquote_plus(value).strip()
+            except ValueError:
+                logger.warning(f"baggage entry format error: {entry}")
+                continue
+            BaggageContext.set_baggage(key, value)
+
+    def inject(self, carrier: Carrier, context: TraceContext):
+        """
+        Inject the trace context into the carrier.
+        Args:
+            carrier: The carrier to inject the trace context into.
+            context: The trace context to inject.
+        """
+        baggage = BaggageContext.get_baggage()
+        if baggage:
+            baggage_header = ",".join(
+                f"{quote_plus(key)}={quote_plus(value)}" for key, value in baggage.items())
+            carrier.set(self._BAGGAGE_HEADER_NAME, baggage_header)

chapter9/gaia-experience/AWorld/aworld/trace/base.py

@@ -0,0 +1,422 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Any, Iterator, Union, Sequence, Protocol, Iterable
+from enum import Enum
+from weakref import WeakSet
+from dataclasses import dataclass, field
+from aworld.logs.util import trace_logger
+
+
+class TraceProvider(ABC):
+
+    @abstractmethod
+    def get_tracer(
+            self,
+            name: str,
+            version: Optional[str] = None
+    ) -> "Tracer":
+        """Returns a `Tracer` for use by the given name.
+
+        This function may return different `Tracer` types (e.g. a no-op tracer
+        vs.  a functional tracer).
+
+        Args:
+            name: The uniquely identifiable name for instrumentation
+                scope, such as instrumentation library, package, module or class name.
+                ``__name__`` may not be used as this can result in
+                different tracer names if the tracers are in different files.
+                It is better to use a fixed string that can be imported where
+                needed and used consistently as the name of the tracer.
+
+                This should *not* be the name of the module that is
+                instrumented but the name of the module doing the instrumentation.
+                E.g., instead of ``"requests"``, use
+                ``"opentelemetry.instrumentation.requests"``.
+
+            version: Optional. The version string of the
+                instrumenting library.  Usually this should be the same as
+                ``importlib.metadata.version(instrumenting_library_name)``
+        """
+
+    @abstractmethod
+    def shutdown(self) -> None:
+        """Shuts down the provider and all its resources.
+        This method should be called when the application is shutting down.
+        """
+
+    @abstractmethod
+    def force_flush(self, timeout: Optional[float] = None) -> bool:
+        """Forces all the data to be sent to the backend.
+        This method should be called when the application is shutting down.
+        Args:
+            timeout: The maximum time to wait for the data to be sent.
+        Returns:
+            True if the data was sent successfully, False otherwise.
+        """
+
+    @abstractmethod
+    def get_current_span(self) -> Optional["Span"]:
+        """Returns the current span from the current context.
+        Returns:
+            The current span from the current context.
+        """
+
+
+class SpanType(Enum):
+    """Specifies additional details on how this span relates to its parent span.
+    """
+
+    #: Default value. Indicates that the span is used internally in the
+    # application.
+    INTERNAL = 0
+
+    #: Indicates that the span describes an operation that handles a remote
+    # request.
+    SERVER = 1
+
+    #: Indicates that the span describes a request to some remote service.
+    CLIENT = 2
+
+    #: Indicates that the span describes a producer sending a message to a
+    #: broker. Unlike client and server, there is usually no direct critical
+    #: path latency relationship between producer and consumer spans.
+    PRODUCER = 3
+
+    #: Indicates that the span describes a consumer receiving a message from a
+    #: broker. Unlike client and server, there is usually no direct critical
+    #: path latency relationship between producer and consumer spans.
+    CONSUMER = 4
+
+
+AttributeValueType = Union[
+    str,
+    bool,
+    int,
+    float,
+    Sequence[str],
+    Sequence[bool],
+    Sequence[int],
+    Sequence[float],
+]
+
+
+class Tracer(ABC):
+    """Handles span creation and in-process context propagation.
+    """
+
+    @abstractmethod
+    def start_span(
+            self,
+            name: str,
+            span_type: SpanType = SpanType.INTERNAL,
+            attributes: dict[str, AttributeValueType] = None,
+            start_time: Optional[int] = None,
+            record_exception: bool = True,
+            set_status_on_exception: bool = True,
+            trace_context: Optional["TraceContext"] = None,
+    ) -> "Span":
+        """Starts and returns a new Span.
+        Args:
+            name: The name of the span.
+            kind: The span's kind (relationship to parent). Note that is
+                meaningful even if there is no parent.
+            attributes: The span's attributes.
+            start_time: Sets the start time of a span
+            record_exception: Whether to record any exceptions raised within the
+                context as error event on the span.
+            set_status_on_exception: Only relevant if the returned span is used
+                in a with/context manager. Defines whether the span status will
+                be automatically set to ERROR when an uncaught exception is
+                raised in the span with block. The span status won't be set by
+                this mechanism if it was previously set manually.
+            trace_context: The trace context to use for the span. If not
+                provided, the current trace context will be used.
+        """
+
+    @abstractmethod
+    def start_as_current_span(
+            self,
+            name: str,
+            span_type: SpanType = SpanType.INTERNAL,
+            attributes: dict[str, AttributeValueType] = None,
+            start_time: Optional[int] = None,
+            record_exception: bool = True,
+            set_status_on_exception: bool = True,
+            end_on_exit: bool = True,
+            trace_context: Optional['TraceContext'] = None
+    ) -> Iterator["Span"]:
+        """Context manager for creating a new span and set it
+        as the current span in this tracer's context.
+
+        Example::
+
+            with tracer.start_as_current_span("one") as parent:
+                parent.add_event("parent's event")
+                with tracer.start_as_current_span("two") as child:
+                    child.add_event("child's event")
+                    trace.get_current_span()  # returns child
+                trace.get_current_span()      # returns parent
+            trace.get_current_span()          # returns previously active span
+
+            This can also be used as a decorator::
+            @tracer.start_as_current_span("name")
+            def function():
+
+        Args:
+            name: The name of the span to be created.
+            kind: The span's kind (relationship to parent). Note that is
+                meaningful even if there is no parent.
+            attributes: The span's attributes.
+            start_time: Sets the start time of a span
+            record_exception: Whether to record any exceptions raised within the
+                context as error event on the span.
+            set_status_on_exception: Only relevant if the returned span is used
+                in a with/context manager. Defines whether the span status will
+                be automatically set to ERROR when an uncaught exception is
+                raised in the span with block. The span status won't be set by
+                this mechanism if it was previously set manually.
+            end_on_exit: Whether to end the span automatically when leaving the
+                context manager.
+            trace_context: The trace context to use for the span. If not
+                provided, the current trace context will be used.
+    """
+
+
+class Span(ABC):
+    """A Span represents a single operation within a trace.
+    """
+
+    @abstractmethod
+    def end(self, end_time: Optional[int] = None) -> None:
+        """Sets the current time as the span's end time.
+
+        The span's end time is the wall time at which the operation finished.
+
+        Only the first call to `end` should modify the span, and
+        implementations are free to ignore or raise on further calls.
+        """
+
+    @abstractmethod
+    def set_attribute(self, key: str, value: Any) -> None:
+        """Sets an attribute on the Span.
+        Args:
+            key: The attribute key.
+            value: The attribute value.
+        """
+
+    @abstractmethod
+    def set_attributes(self, attributes: dict[str, Any]) -> None:
+        """Sets multiple attributes on the Span.
+        Args:
+            attributes: A dictionary of attributes to set.
+        """
+
+    @abstractmethod
+    def is_recording(self) -> bool:
+        """Returns whether this span will be recorded.
+        Returns true if this Span is active and recording information like attributes using set_attribute.
+        """
+
+    @abstractmethod
+    def record_exception(
+            self,
+            exception: BaseException,
+            attributes: dict[str, Any] = None,
+            timestamp: Optional[int] = None,
+            escaped: bool = False,
+    ) -> None:
+        """Records an exception in the span.
+        Args:
+            exception: The exception to record.
+            attributes: A dictionary of attributes to set on the exception event.
+            timestamp: The timestamp of the exception.
+            escaped: Whether the exception was escaped.
+        """
+
+    @abstractmethod
+    def get_trace_id(self) -> str:
+        """Returns the trace ID of the span.
+        Returns:
+            The trace ID of the span.
+        """
+
+    @abstractmethod
+    def get_span_id(self) -> str:
+        """Returns the ID of the span.
+        Returns:
+            The ID of the span.
+        """
+
+    def _add_to_open_spans(self) -> None:
+        """Add the current span to OPEN_SPANS."""
+        _OPEN_SPANS.add(self)
+
+    def _remove_from_open_spans(self) -> None:
+        """Remove the current span from OPEN_SPANS."""
+        _OPEN_SPANS.discard(self)
+
+
+class NoOpSpan(Span):
+    """No-op implementation of `Span`."""
+
+    def end(self, end_time: Optional[int] = None) -> None:
+        pass
+
+    def set_attribute(self, key: str, value: Any) -> None:
+        pass
+
+    def set_attributes(self, attributes: dict[str, Any]) -> None:
+        pass
+
+    def is_recording(self) -> bool:
+        return False
+
+    def record_exception(
+            self,
+            exception: BaseException,
+            attributes: dict[str, Any] = None,
+            timestamp: Optional[int] = None,
+            escaped: bool = False,
+    ) -> None:
+        pass
+
+    def get_trace_id(self) -> str:
+        return ""
+
+    def get_span_id(self) -> str:
+        return ""
+
+
+class NoOpTracer(Tracer):
+    """No-op implementation of `Tracer`."""
+
+    def start_span(
+            self,
+            name: str,
+            span_type: SpanType = SpanType.INTERNAL,
+            attributes: dict[str, AttributeValueType] = None,
+            start_time: Optional[int] = None,
+            record_exception: bool = True,
+            set_status_on_exception: bool = True,
+            trace_context: Optional["TraceContext"] = None,
+    ) -> Span:
+        return NoOpSpan()
+
+    def start_as_current_span(
+            self,
+            name: str,
+            span_type: SpanType = SpanType.INTERNAL,
+            attributes: dict[str, AttributeValueType] = None,
+            start_time: Optional[int] = None,
+            record_exception: bool = True,
+            set_status_on_exception: bool = True,
+            end_on_exit: bool = True,
+            trace_context: Optional['TraceContext'] = None
+    ) -> Iterator[Span]:
+        yield NoOpSpan()
+
+
+class Carrier(Protocol):
+    """Carrier is a protocol that represents a carrier for trace context.
+    """
+
+    def get(self, key: str) -> Optional[str]:
+        """Returns the value of the given key from the carrier.
+        Args:
+            key: The key to get the value for.
+        Returns:
+            The value of the given key from the carrier.
+        """
+
+    def set(self, key: str, value: str) -> None:
+        """Sets the value of the given key in the carrier.
+        Args:
+            key: The key to set the value for.
+            value: The value to set.
+        """
+
+    def keys(self) -> Iterable[str]:
+        """Returns an iterable of keys in the carrier.
+        Returns:
+            An iterable of keys in the carrier.
+        """
+
+
+@dataclass(frozen=True)
+class TraceContext:
+    """TraceContext is a class that represents a trace context.
+    """
+    trace_id: str
+    span_id: str
+    version: str = "00"
+    trace_flags: str = "01"
+    attributes: dict[str, Any] = field(default_factory=dict)
+
+
+class Propagator(ABC):
+    """Propagator is a protocol that represents a propagator for trace context.
+    """
+
+    def _get_value(self, carrier: Carrier, name: str) -> str:
+        """
+        Get value from carrier.
+        Args:
+            carrier: The carrier to get value from.
+            name: The name of the value.
+        Returns:
+            The value of the name.
+        """
+        return carrier.get(name) or carrier.get('HTTP_' + name.upper().replace('-', '_'))
+
+    @abstractmethod
+    def extract(self, carrier: Carrier) -> Optional[TraceContext]:
+        """Extracts a trace context from the given carrier.
+        Args:
+            carrier: The carrier to extract the trace context from.
+        Returns:
+            The trace context extracted from the carrier.
+        """
+    @abstractmethod
+    def inject(self, trace_context: TraceContext, carrier: Carrier) -> None:
+        """Injects a trace context into the given carrier.
+        Args:
+            trace_context: The trace context to inject.
+            carrier: The carrier to inject the trace context into.
+        """
+
+
+_GLOBAL_TRACER_PROVIDER: Optional[TraceProvider] = None
+_OPEN_SPANS: WeakSet[Span] = WeakSet()
+
+
+def set_tracer_provider(provider: TraceProvider):
+    """
+    Set the global tracer provider.
+    """
+    global _GLOBAL_TRACER_PROVIDER
+    _GLOBAL_TRACER_PROVIDER = provider
+
+
+def get_tracer_provider() -> TraceProvider:
+    """
+    Get the global tracer provider.
+    """
+    global _GLOBAL_TRACER_PROVIDER
+    if _GLOBAL_TRACER_PROVIDER is None:
+        raise Exception("No tracer provider has been set.")
+    return _GLOBAL_TRACER_PROVIDER
+
+
+def get_tracer_provider_silent():
+    try:
+        return get_tracer_provider()
+    except Exception:
+        return None
+
+
+def log_trace_error():
+    """
+    Log an error with traceback information.
+    """
+    trace_logger.exception(
+        'This is logging the trace internal error.',
+    )

chapter9/gaia-experience/AWorld/aworld/trace/config.py

@@ -0,0 +1,101 @@
+import os
+from pydantic import BaseModel, ConfigDict
+from typing import Sequence, Optional
+from aworld.trace.span_cosumer import SpanConsumer
+from logging import Logger
+from aworld.logs.util import trace_logger
+from aworld.trace.context_manager import trace_configure
+from aworld.metrics.context_manager import MetricContext
+from aworld.logs.log import set_log_provider, instrument_logging
+from aworld.trace.instrumentation.uni_llmmodel import LLMModelInstrumentor
+from aworld.trace.instrumentation.eventbus import EventBusInstrumentor
+from aworld.trace.instrumentation.agent import AgentInstrumentor
+from aworld.trace.instrumentation.tool import ToolInstrumentor
+
+from aworld.trace.opentelemetry.memory_storage import TraceStorage
+
+
+class ObservabilityConfig(BaseModel):
+    '''
+    Observability configuration
+    '''
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    trace_provider: Optional[str] = "otlp"
+    trace_backends: Optional[Sequence[str]] = ["memory"]
+    trace_base_url: Optional[str] = None
+    trace_write_token: Optional[str] = None
+    trace_span_consumers: Optional[Sequence[SpanConsumer]] = None
+    trace_storage: Optional[TraceStorage] = None
+    # whether to start the trace service
+    trace_server_enabled: Optional[bool] = False
+    trace_server_port: Optional[int] = 7079
+    metrics_provider: Optional[str] = None
+    metrics_backend: Optional[str] = None
+    metrics_base_url: Optional[str] = None
+    metrics_write_token: Optional[str] = None
+    # whether to instrument system metrics
+    metrics_system_enabled: Optional[bool] = False
+    logs_provider: Optional[str] = None
+    logs_backend: Optional[str] = None
+    logs_base_url: Optional[str] = None
+    logs_write_token: Optional[str] = None
+    # The loggers that need to record the log as a span
+    logs_trace_instrumented_loggers: Sequence[Logger] = [trace_logger]
+
+
+def configure(config: ObservabilityConfig = None):
+    if config is None:
+        config = ObservabilityConfig()
+    _trace_configure(config)
+    _metrics_configure(config)
+    _log_configure(config)
+    LLMModelInstrumentor().instrument()
+    EventBusInstrumentor().instrument()
+    AgentInstrumentor().instrument()
+    ToolInstrumentor().instrument()
+
+
+def _trace_configure(config: ObservabilityConfig):
+    if not config.trace_base_url and config.trace_provider == "otlp":
+        if "logfire" in config.trace_backends:
+            config.trace_base_url = os.getenv("LOGFIRE_WRITE_TOKEN")
+        elif os.getenv("OTLP_TRACES_ENDPOINT"):
+            config.trace_base_url = os.getenv("OTLP_TRACES_ENDPOINT")
+            config.trace_backends.append("other_otlp")
+
+    trace_configure(
+        provider=config.trace_provider,
+        backends=config.trace_backends,
+        base_url=config.trace_base_url,
+        write_token=config.trace_write_token,
+        span_consumers=config.trace_span_consumers,
+        server_enabled=config.trace_server_enabled,
+        server_port=config.trace_server_port,
+        storage=config.trace_storage
+    )
+
+
+def _metrics_configure(config: ObservabilityConfig):
+    if config.metrics_provider and config.metrics_backend:
+        MetricContext.configure(
+            provider=config.metrics_provider,
+            backend=config.metrics_backend,
+            base_url=config.metrics_base_url,
+            write_token=config.metrics_write_token,
+            metrics_system_enabled=config.metrics_system_enabled
+        )
+
+
+def _log_configure(config: ObservabilityConfig):
+    if config.logs_provider and config.logs_backend:
+        if config.logs_backend == "logfire" and not config.logs_write_token:
+            config.logs_write_token = os.getenv("LOGFIRE_WRITE_TOKEN")
+        set_log_provider(provider=config.logs_provider,
+                         backend=config.logs_backend,
+                         base_url=config.logs_base_url,
+                         write_token=config.logs_write_token)
+
+    if config.logs_trace_instrumented_loggers:
+        for logger in config.logs_trace_instrumented_loggers:
+            instrument_logging(logger)

chapter9/gaia-experience/AWorld/aworld/trace/constants.py

@@ -0,0 +1,57 @@
+from enum import Enum
+
+ATTRIBUTES_NAMESPACE = 'aworld'
+"""Namespace within OTEL attributes used by aworld."""
+
+ATTRIBUTES_MESSAGE_KEY = f'{ATTRIBUTES_NAMESPACE}.msg'
+"""The formatted message for a log."""
+
+ATTRIBUTES_MESSAGE_TEMPLATE_KEY = f'{ATTRIBUTES_NAMESPACE}.msg_template'
+"""The template for a log message."""
+
+ATTRIBUTES_MESSAGE_RUN_TYPE_KEY = f'{ATTRIBUTES_NAMESPACE}.run_type'
+"""The template for a log message."""
+
+MESSAGE_FORMATTED_VALUE_LENGTH_LIMIT = 128
+"""Maximum number of characters for formatted values in a trace message."""
+
+SPAN_NAME_PREFIX_EVENT = "event."
+"""Prefix for event span name."""
+
+SPAN_NAME_PREFIX_EVENT_AGENT = SPAN_NAME_PREFIX_EVENT + "agent."
+"""Prefix for event span name of agent."""
+
+SPAN_NAME_PREFIX_EVENT_TOOL = SPAN_NAME_PREFIX_EVENT + "tool."
+"""Prefix for event span name of tool."""
+
+SPAN_NAME_PREFIX_EVENT_TASK = SPAN_NAME_PREFIX_EVENT + "task."
+"""Prefix for event span name of task."""
+
+SPAN_NAME_PREFIX_EVENT_OUTPUT = SPAN_NAME_PREFIX_EVENT + "output."
+"""Prefix for event span name of output."""
+
+SPAN_NAME_PREFIX_EVENT_OTHER = SPAN_NAME_PREFIX_EVENT + "other."
+"""Prefix for event span name of error."""
+
+SPAN_NAME_PREFIX_TASK = "task."
+"""Prefix for task span name."""
+
+SPAN_NAME_PREFIX_AGENT = "agent."
+"""Prefix for agent span name."""
+
+SPAN_NAME_PREFIX_TOOL = "tool."
+"""Prefix for tool span name."""
+
+SPAN_NAME_PREFIX_LLM = "llm."
+"""Prefix for llm span name."""
+
+
+class RunType(Enum):
+    '''Span run type supported in the framework
+    '''
+    AGNET = "AGENT"
+    TOOL = "TOOL"
+    MCP = "MCP"
+    LLM = "LLM"
+    TASK = "TASK"
+    OTHER = "OTHER"

chapter9/gaia-experience/AWorld/aworld/trace/context_manager.py

@@ -0,0 +1,316 @@
+import types
+import inspect
+from typing import Union, Optional, Any, Type, Sequence, Callable, Iterable
+from aworld.trace.base import (
+    AttributeValueType,
+    NoOpSpan,
+    Span, Tracer,
+    NoOpTracer,
+    get_tracer_provider,
+    get_tracer_provider_silent,
+    log_trace_error
+)
+from aworld.trace.span_cosumer import SpanConsumer
+from aworld.version_gen import __version__
+from aworld.trace.auto_trace import AutoTraceModule, install_auto_tracing
+from aworld.trace.stack_info import get_user_stack_info
+from aworld.trace.constants import (
+    ATTRIBUTES_MESSAGE_KEY,
+    ATTRIBUTES_MESSAGE_RUN_TYPE_KEY,
+    ATTRIBUTES_MESSAGE_TEMPLATE_KEY,
+    RunType
+)
+from aworld.trace.msg_format import (
+    chunks_formatter,
+    warn_formatting,
+    FStringAwaitError,
+    KnownFormattingError,
+    warn_fstring_await
+)
+from aworld.trace.function_trace import trace_func
+from .opentelemetry.opentelemetry_adapter import configure_otlp_provider
+from aworld.logs.util import logger
+
+
+def trace_configure(provider: str = "otlp",
+                    backends: Sequence[str] = None,
+                    base_url: str = None,
+                    write_token: str = None,
+                    span_consumers: Optional[Sequence[SpanConsumer]] = None,
+                    **kwargs
+                    ) -> None:
+    """
+    Configure the trace provider.
+    Args:
+        provider: The trace provider to use.
+        backends: The trace backends to use.
+        base_url: The base URL of the trace backend.
+        write_token: The write token of the trace backend.
+        span_consumers: The span consumers to use.
+        **kwargs: Additional arguments to pass to the trace provider.
+    Returns:
+        None
+    """
+    exist_provider = get_tracer_provider_silent()
+    if exist_provider:
+        logger.info("Trace provider already configured, shutting down...")
+        exist_provider.shutdown()
+    if provider == "otlp":
+        configure_otlp_provider(
+            backends=backends, base_url=base_url, write_token=write_token, span_consumers=span_consumers, **kwargs)
+    else:
+        raise ValueError(f"Unknown trace provider: {provider}")
+
+
+class TraceManager:
+    """
+    TraceManager is a class that provides a way to trace the execution of a function.
+    """
+
+    def __init__(self, tracer_name: str = None) -> None:
+        self._tracer_name = tracer_name or "aworld"
+        self._version = __version__
+
+    def _create_auto_span(self,
+                          name: str,
+                          attributes: dict[str, AttributeValueType] = None
+                          ) -> Span:
+        """
+        Create a auto trace span with the given name and attributes.
+        """
+        return self._create_context_span(name, attributes)
+
+    def _create_context_span(self,
+                             name: str,
+                             attributes: dict[str, AttributeValueType] = None) -> Span:
+        try:
+            tracer = get_tracer_provider().get_tracer(
+                name=self._tracer_name, version=self._version)
+            return ContextSpan(span_name=name, tracer=tracer, attributes=attributes)
+        except Exception:
+            return ContextSpan(span_name=name, tracer=NoOpTracer(), attributes=attributes)
+
+    def get_current_span(self) -> Span:
+        """
+        Get the current span.
+        """
+        try:
+            return get_tracer_provider().get_current_span()
+        except Exception:
+            return NoOpSpan()
+
+    def new_manager(self, tracer_name_suffix: str = None) -> "TraceManager":
+        """
+        Create a new TraceManager with the given tracer name suffix.
+        """
+        tracer_name = self._tracer_name if not tracer_name_suffix else f"{self._tracer_name}.{tracer_name_suffix}"
+        return TraceManager(tracer_name=tracer_name)
+
+    def auto_tracing(self,
+                     modules: Union[Sequence[str], Callable[[AutoTraceModule], bool]],
+                     min_duration: float) -> None:
+        """
+        Automatically trace the execution of a function.
+        Args:
+            modules: A list of module names or a callable that takes a `AutoTraceModule` and returns a boolean.
+            min_duration: The minimum duration of a function to be traced.
+        Returns:
+            None
+        """
+        install_auto_tracing(self, modules, min_duration)
+
+    def span(self,
+             msg_template: str = "",
+             attributes: dict[str, AttributeValueType] = None,
+             *,
+             span_name: str = None,
+             run_type: RunType = RunType.OTHER) -> "ContextSpan":
+
+        try:
+            attributes = attributes or {}
+            stack_info = get_user_stack_info()
+            merged_attributes = {**stack_info, **attributes}
+            # Retrieve stack information of user code and add it to the attributes
+
+            if any(c in msg_template for c in ('{', '}')):
+                fstring_frame = inspect.currentframe().f_back
+            else:
+                fstring_frame = None
+            log_message, extra_attrs, msg_template = format_span_msg(
+                msg_template,
+                merged_attributes,
+                fstring_frame=fstring_frame,
+            )
+            merged_attributes[ATTRIBUTES_MESSAGE_KEY] = log_message
+            merged_attributes.update(extra_attrs)
+            merged_attributes[ATTRIBUTES_MESSAGE_TEMPLATE_KEY] = msg_template
+            merged_attributes[ATTRIBUTES_MESSAGE_RUN_TYPE_KEY] = run_type.value
+            span_name = span_name or msg_template
+
+            return self._create_context_span(span_name, merged_attributes)
+
+        except Exception:
+            log_trace_error()
+            return ContextSpan(span_name=span_name, tracer=NoOpTracer(), attributes=attributes)
+
+    def func_span(self,
+                  msg_template: Union[str, Callable] = None,
+                  *,
+                  attributes: dict[str, AttributeValueType] = None,
+                  span_name: str = None,
+                  extract_args: Union[bool, Iterable[str]] = False,
+                  **kwargs) -> Callable:
+        """
+        A decorator that traces the execution of a function.
+        Args:
+            msg_template: The message template to use.
+            attributes: The attributes to add to the span.
+            span_name: The name of the span.
+            extract_args: Whether to extract arguments from the function call.
+            **kwargs: Additional attributes to add to the span.
+        Returns:
+            A decorator that traces the execution of a function.
+        """
+        if callable(msg_template):
+            #  @trace_func
+            #  def foo():
+            return self.func_span()(msg_template)
+
+        attributes = attributes or {}
+        attributes.update(kwargs)
+        return trace_func(self, msg_template, attributes, span_name, extract_args)
+
+
+class ContextSpan(Span):
+    """A context manager that wraps an existing `Span` object.
+    This class provides a way to use a `Span` object as a context manager.
+    When the context manager is entered, it returns the `Span` itself.
+    When the context manager is exited, it calls `end` on the `Span`.
+    Args:
+        span: The `Span` object to wrap.
+    """
+
+    def __init__(self,
+                 span_name: str,
+                 tracer: Tracer,
+                 attributes: dict[str, AttributeValueType] = None) -> None:
+        self._span_name = span_name
+        self._tracer = tracer
+        self._attributes = attributes
+        self._span: Span = None
+        self._coro_context = None
+
+    def _start(self):
+        if self._span is not None:
+            return
+
+        self._span = self._tracer.start_span(
+            name=self._span_name,
+            attributes=self._attributes,
+        )
+
+    def __enter__(self) -> "Span":
+        self._start()
+        return self
+
+    def __exit__(
+            self,
+            exc_type: Optional[Type[BaseException]],
+            exc_val: Optional[BaseException],
+            traceback: Optional[Any],
+    ) -> None:
+        """Ends context manager and calls `end` on the `Span`."""
+        self._handle_exit(exc_type, exc_val, traceback)
+
+    async def __aenter__(self) -> "Span":
+        self._start()
+
+        return self
+
+    async def __aexit__(
+            self,
+            exc_type: Optional[Type[BaseException]],
+            exc_val: Optional[BaseException],
+            traceback: Optional[Any],
+    ) -> None:
+        self._handle_exit(exc_type, exc_val, traceback)
+
+    def _handle_exit(
+            self,
+            exc_type: Optional[Type[BaseException]],
+            exc_val: Optional[BaseException],
+            traceback: Optional[Any],
+    ) -> None:
+        try:
+            if self._span and self._span.is_recording() and isinstance(exc_val, BaseException):
+                self._span.record_exception(exc_val, escaped=True)
+        except ValueError as e:
+            logger.warning(f"Failed to record_exception: {e}")
+        finally:
+            if self._span:
+                self._span.end()
+
+    def end(self, end_time: Optional[int] = None) -> None:
+        if self._span:
+            self._span.end(end_time)
+
+    def set_attribute(self, key: str, value: AttributeValueType) -> None:
+        if self._span:
+            self._span.set_attribute(key, value)
+
+    def set_attributes(self, attributes: dict[str, AttributeValueType]) -> None:
+        if self._span:
+            self._span.set_attributes(attributes)
+
+    def is_recording(self) -> bool:
+        if self._span:
+            return self._span.is_recording()
+        return False
+
+    def record_exception(
+            self,
+            exception: BaseException,
+            attributes: dict[str, Any] = None,
+            timestamp: Optional[int] = None,
+            escaped: bool = False,
+    ) -> None:
+        if self._span:
+            self._span.record_exception(
+                exception, attributes, timestamp, escaped)
+
+    def get_trace_id(self) -> str:
+        if self._span:
+            return self._span.get_trace_id()
+
+    def get_span_id(self) -> str:
+        if self._span:
+            return self._span.get_span_id()
+
+
+def format_span_msg(
+        format_string: str,
+        kwargs: dict[str, Any],
+        fstring_frame: types.FrameType = None,
+) -> tuple[str, dict[str, Any], str]:
+    """ Returns
+    1. The formatted message.
+    2. A dictionary of extra attributes to add to the span/log.
+         These can come from evaluating values in f-strings.
+    3. The final message template, which may differ from `format_string` if it was an f-string.
+    """
+    try:
+        chunks, extra_attrs, new_template = chunks_formatter.chunks(
+            format_string,
+            kwargs,
+            fstring_frame=fstring_frame
+        )
+        return ''.join(chunk['v'] for chunk in chunks), extra_attrs, new_template
+    except KnownFormattingError as e:
+        warn_formatting(str(e) or str(e.__cause__))
+    except FStringAwaitError as e:
+        warn_fstring_await(str(e))
+    except Exception:
+        log_trace_error()
+
+    # Formatting failed, so just use the original format string as the message.
+    return format_string, {}, format_string

chapter9/gaia-experience/AWorld/aworld/trace/function_trace.py

@@ -0,0 +1,180 @@
+import inspect
+import contextlib
+import functools
+import json
+from typing import TYPE_CHECKING, Callable, Any, Union, Iterable, Sequence
+from aworld.trace.base import (
+    AttributeValueType
+)
+
+from aworld.trace.stack_info import get_filepath_attribute
+from aworld.trace.constants import (
+    ATTRIBUTES_MESSAGE_TEMPLATE_KEY
+)
+from aworld.utils.serialized_util import to_serializable
+
+if TYPE_CHECKING:
+    from aworld.trace.context_manager import TraceManager, ContextSpan
+
+
+def trace_func(trace_manager: "TraceManager",
+               msg_template: str = None,
+               attributes: dict[str, AttributeValueType] = None,
+               span_name: str = None,
+               extract_args: Union[bool, Iterable[str]] = False):
+    """A decorator that traces the execution of a function.
+
+    Args:
+        trace_manager: The trace manager to use.
+        msg_template: The message template to use.
+        attributes: The attributes to use.
+        span_name: The span name to use.
+        extract_args: Whether to extract arguments from the function call.
+
+    Returns:
+        The decorated function.
+    """
+
+    def decorator(func: Callable) -> Callable:
+        func_meta = get_function_meta(func, msg_template)
+        func_meta.update(attributes or {})
+        final_span_name = span_name or func_meta.get(ATTRIBUTES_MESSAGE_TEMPLATE_KEY) or func.__name__
+
+        if inspect.isgeneratorfunction(func):
+            def wrapper(*args, **kwargs):
+                with open_func_span(trace_manager, func_meta, final_span_name,
+                                    get_func_args(func, extract_args, *args, **kwargs)):
+                    for item in func(*args, **kwargs):
+                        yield item
+        elif inspect.isasyncgenfunction(func):
+            async def wrapper(*args, **kwargs):
+                with open_func_span(trace_manager, func_meta, final_span_name,
+                                    get_func_args(func, extract_args, *args, **kwargs)):
+                    async for item in func(*args, **kwargs):
+                        yield item
+        elif inspect.iscoroutinefunction(func):
+            async def wrapper(*args, **kwargs):
+                with open_func_span(trace_manager, func_meta, final_span_name,
+                                    get_func_args(func, extract_args, *args, **kwargs)):
+                    return await func(*args, **kwargs)
+        else:
+            def wrapper(*args, **kwargs):
+                with open_func_span(trace_manager, func_meta, final_span_name,
+                                    get_func_args(func, extract_args, *args, **kwargs)):
+                    return func(*args, **kwargs)
+
+        wrapper = functools.wraps(func)(wrapper)  # type: ignore
+        return wrapper
+
+    return decorator
+
+
+def open_func_span(trace_manager: "TraceManager",
+                   func_meta: dict[str, AttributeValueType],
+                   span_name: str,
+                   func_args: dict[str, AttributeValueType]):
+    """Open a function span.
+
+    Args:
+        func_meta: The function meta information.
+        span_name: The span name.
+
+    Returns:
+        The function span.
+    """
+    func_meta.update(func_args)
+    return trace_manager._create_auto_span(name=span_name, attributes=func_meta)
+
+
+def get_func_args(func: Callable,
+                  extract_args: Union[bool, Iterable[str]] = False,
+                  *args,
+                  **kwargs):
+    """Get the arguments of a function.
+
+    Args:
+        func: The function to get the arguments of.
+        extract_args: Whether to extract arguments from the function call.
+        *args: The positional arguments.
+        **kwargs: The keyword arguments.
+
+    Returns:
+        The arguments of the function.
+    """
+    func_sig = inspect.signature(func)
+    if func_sig.parameters:
+        func_args = func_sig.bind(*args, **kwargs).arguments
+        if extract_args is not False:
+            if isinstance(extract_args, bool):
+                extract_args = func_sig.parameters.keys()
+            func_args = {k: v for k, v in func_args.items() if k in extract_args}
+        pre_process_func_args(func_args)
+        return func_args
+    return {}
+
+
+def pre_process_func_args(args: dict):
+    """Pre process the function arguments.
+    """
+    if "self" in args:
+        args.pop("self")
+    for k, v in args.items():
+        if (v and not isinstance(v, (str, bool, int, float)) and
+                not (isinstance(v, Sequence) and all(isinstance(i, (str, bool, int, float)) for i in v))):
+            args[k] = json.dumps(to_serializable(v), ensure_ascii=False)
+
+
+def get_function_meta(func: Any,
+                      msg_template: str = None) -> dict[str, AttributeValueType]:
+    """Get the meta information of a function.\
+
+    Args:
+        func: The function to get the meta information of.
+        msg_template: The message template to use.
+
+    Returns:
+        The meta information of the function.
+    """
+    func = inspect.unwrap(func)
+    if not inspect.isfunction(func) and hasattr(func, '__call__'):
+        func = func.__call__
+        func = inspect.unwrap(func)
+
+    func_name = getattr(func, '__qualname__', getattr(func, '__name__', build_func_name(func)))
+    if not msg_template:
+        try:
+            msg_template = f'Calling {inspect.getmodule(func).__name__}.{func_name}'  # type: ignore
+        except Exception:  # pragma: no cover
+            msg_template = f'Calling {func_name}'
+    meta: dict[str, AttributeValueType] = {
+        'code.function': func_name,
+        ATTRIBUTES_MESSAGE_TEMPLATE_KEY: msg_template,
+    }
+    with contextlib.suppress(Exception):
+        meta['code.lineno'] = func.__code__.co_firstlineno
+    with contextlib.suppress(Exception):
+        # get code.filepath
+        meta.update(get_filepath_attribute(inspect.getsourcefile(func)))
+
+    func_sig = inspect.signature(func)
+    if func_sig.parameters:
+        meta['func.args'] = [str(param) for param in func_sig.parameters.values()
+                             if param.name != 'self']
+    return meta
+
+
+def build_func_name(func: Any) -> str:
+    """Build the function name.
+
+    Args:
+        func: The function to build the name of.
+
+    Returns:
+        The function name.
+    """
+    try:
+        result = repr(func)
+    except Exception:
+        result = f'<{type(func).__name__} object>'
+
+    return result

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/__init__.py

@@ -0,0 +1,87 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+from abc import ABC, abstractmethod
+from typing import Any, Collection
+from importlib_metadata import version, PackageNotFoundError
+from aworld.logs.util import logger
+from aworld.utils.import_package import import_package
+import_package("packaging")  # noqa
+from packaging.requirements import Requirement, InvalidRequirement
+
+
+class Instrumentor(ABC):
+    _instance = None
+    _has_instrumented = False
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = object.__new__(cls)
+
+        return cls._instance
+
+    def instrument(self, **kwargs: Any):
+        """
+        Instrument the library.
+        """
+        if self._has_instrumented:
+            logger.warning(
+                f"Instrumentor[{self.__class__.__name__}] has already instrumented, skip")
+            return
+
+        if not self._check_dependency_conflicts():
+            return
+
+        result = self._instrument(**kwargs)
+        self._has_instrumented = True
+        return result
+
+    def uninstrument(self, **kwargs: Any):
+        """
+        Uninstrument the library.
+        """
+        if not self._has_instrumented:
+            logger.warning("Instrumentor has not instrumented, skip")
+            return
+        self._uninstrument(**kwargs)
+        self._has_instrumented = False
+
+    @abstractmethod
+    def _uninstrument(self, **kwargs: Any):
+        """
+        Uninstrument the library.
+        """
+
+    @abstractmethod
+    def _instrument(self, **kwargs: Any):
+        """
+        Instrument the library.
+        """
+
+    def _check_dependency_conflicts(self):
+        dependencies = self.instrumentation_dependencies()
+        for dependence in dependencies:
+            try:
+                requirement = Requirement(dependence)
+            except InvalidRequirement as exc:
+                logger.warning(
+                    f'error parsing dependency, reporting as a conflict: "{dependence}" - {exc}')
+                return False
+            try:
+                dist_version = version(requirement.name)
+            except PackageNotFoundError as exc:
+                logger.warning(
+                    f'dependency not found, reporting as a conflict: "{dependence}" - {exc}')
+                return False
+
+            if requirement.specifier and not requirement.specifier.contains(dist_version):
+                logger.warning(
+                    f'dependency version conflict, reporting as a conflict: requested: "{self.required}" but found: "{self.found}"')
+                return False
+
+        return True
+
+    @abstractmethod
+    def instrumentation_dependencies(self) -> Collection[str]:
+        """
+        Return a list of dependencies that the instrumentation requires.
+        """

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/agent/__init__.py

@@ -0,0 +1,238 @@
+import wrapt
+import time
+import traceback
+import aworld.trace.constants as trace_constants
+from typing import Collection, Any
+from aworld.trace.instrumentation import Instrumentor
+from aworld.trace.instrumentation import semconv
+from aworld.trace.base import (
+    Tracer,
+    SpanType,
+    get_tracer_provider_silent
+)
+from aworld.logs.util import logger
+from aworld.metrics.context_manager import MetricContext
+from aworld.metrics.template import MetricTemplate
+from aworld.metrics.metric import MetricType
+
+agent_duration_histogram = MetricTemplate(
+    type=MetricType.HISTOGRAM,
+    name="agent_run_duration",
+    unit="s",
+    description="Agent run duration",
+)
+
+agent_run_counter = MetricTemplate(
+    type=MetricType.COUNTER,
+    name="agent_run_counter",
+    unit="time",
+    description="Number of agent run or async run",
+)
+
+agent_usage_histogram = MetricTemplate(
+    type=MetricType.HISTOGRAM,
+    name="agent_token_usage",
+    unit="token",
+    description="Agent token usage"
+)
+
+
+def get_agent_span_attributes(instance, message):
+    return {
+        semconv.AGENT_ID: instance.id(),
+        semconv.AGENT_NAME: instance.name(),
+        semconv.TASK_ID: message.context.task_id if (message.context and message.context.task_id) else "",
+        semconv.SESSION_ID: message.context.session_id if (message.context and message.context.session_id) else message.session_id,
+        semconv.USER_ID: message.context.user if (message.context and message.context.user) else "",
+        trace_constants.ATTRIBUTES_MESSAGE_RUN_TYPE_KEY: trace_constants.RunType.AGNET.value
+    }
+
+
+def _end_span(span):
+    if span:
+        span.end()
+
+
+def _record_metric(duration, attributes, exception=None):
+    if MetricContext.metric_initialized():
+        MetricContext.histogram_record(agent_duration_histogram, duration, labels=attributes)
+        if exception:
+            run_counter_attr = {
+                semconv.AGENT_RUN_SUCCESS: "0",
+                "error.type": exception.__class__.__name__,
+                **attributes
+            }
+        else:
+            run_counter_attr = {
+                semconv.AGENT_RUN_SUCCESS: "1",
+                **attributes
+            }
+        MetricContext.count(agent_run_counter, 1, labels=run_counter_attr)
+
+
+def _record_exception(span, start_time, exception, attributes):
+    try:
+        duration = time.time() - start_time if "start_time" in locals() else 0
+        if span and span.is_recording:
+            span.record_exception(exception=exception)
+        _record_metric(duration, attributes, exception)
+    except Exception as e:
+        logger.warning(f"agent instrument record exception error.{e}")
+
+
+def _record_response(instance,
+                     start_time,
+                     response,
+                     attributes):
+    try:
+        duration = time.time() - start_time if "start_time" in locals() else 0
+        _record_metric(duration, attributes)
+        # if instance and instance.agent_context and instance.agent_context.llm_output and MetricContext.metric_initialized():
+        #     usage = instance.agent_context.llm_output.usage
+        #     for usage_type in ["completion_tokens", "prompt_tokens", "total_tokens"]:
+        #         if usage and usage.get(usage_type):
+        #             labels = {
+        #                 **attributes,
+        #                 semconv.AGENT_USAGE_TYPE: usage_type
+        #             }
+        #             MetricContext.histogram_record(
+        #                 agent_usage_histogram,
+        #                 usage.get(usage_type),
+        #                 labels=labels
+        #             )
+    except Exception as e:
+        logger.warning(f"agent instrument record response error.{e}")
+
+
+def _async_run_class_wrapper(tracer: Tracer):
+    async def _async_run_wrapper(wrapped, instance, args, kwargs):
+        span = None
+        message = args[0] or kwargs.get("message")
+        attributes = get_agent_span_attributes(instance, message)
+        if tracer:
+            span = tracer.start_span(
+                name=trace_constants.SPAN_NAME_PREFIX_AGENT + "async_run",
+                span_type=SpanType.SERVER,
+                attributes=attributes
+            )
+        start_time = time.time()
+        try:
+            response = await wrapped(*args, **kwargs)
+            _record_response(instance, start_time, response, attributes)
+        except Exception as e:
+            _record_exception(span=span,
+                              start_time=start_time,
+                              exception=e,
+                              attributes=attributes
+                              )
+            _end_span(span)
+            raise e
+        _end_span(span)
+        return response
+    return _async_run_wrapper
+
+
+async def _async_run_instance_wrapper(tracer: Tracer):
+
+    @wrapt.decorator
+    async def _awrapper(wrapped, instance, args, kwargs):
+        wrapper_func = _async_run_class_wrapper(tracer=tracer)
+        return await wrapper_func(wrapped, instance, args, kwargs)
+
+    return _awrapper
+
+
+def _call_llm_model_class_wrapper(tracer: Tracer):
+    async def _call_llm_model_wrapper(wrapped, instance, args, kwargs):
+        attributes = {
+            semconv.AGENT_ID: instance.id(),
+            semconv.AGENT_NAME: instance.name()
+        }
+        if hasattr(instance, "context") and instance.context:
+            attributes.update({
+                semconv.TASK_ID: instance.context.task_id if (instance.context and instance.context.task_id) else "",
+                semconv.SESSION_ID: instance.context.session_id if (instance.context and instance.context.session_id) else instance.session_id,
+                semconv.USER_ID: instance.context.user if (instance.context and instance.context.user) else ""
+            })
+        try:
+            response = await wrapped(*args, **kwargs)
+            try:
+                usage = response.usage if hasattr(response, "usage") else None
+                for usage_type in ["completion_tokens", "prompt_tokens", "total_tokens"]:
+                    if usage and usage.get(usage_type):
+                        labels = {
+                            **attributes,
+                            semconv.AGENT_USAGE_TYPE: usage_type
+                        }
+                        MetricContext.histogram_record(
+                            agent_usage_histogram,
+                            usage.get(usage_type),
+                            labels=labels
+                        )
+
+            except Exception as e:
+                logger.warning(f"agent instrument record response error.{e}")
+        except Exception as e:
+            raise e
+        return response
+    return _call_llm_model_wrapper
+
+
+async def _call_llm_model_instance_wrapper(tracer: Tracer):
+
+    @wrapt.decorator
+    async def _awrapper(wrapped, instance, args, kwargs):
+        wrapper_func = _call_llm_model_class_wrapper(tracer=tracer)
+        return await wrapper_func(wrapped, instance, args, kwargs)
+
+    return _awrapper
+
+
+class AgentInstrumentor(Instrumentor):
+
+    def instrumentation_dependencies(self) -> Collection[str]:
+        return ()
+
+    def _instrument(self, **kwargs):
+        agent_trace_enabled = kwargs.get("trace_enabled", False)
+        tracer_provider = get_tracer_provider_silent()
+        tracer = None
+        if tracer_provider and agent_trace_enabled:
+            tracer = tracer_provider.get_tracer(
+                "aworld.trace.instrumentation.agent")
+
+        try:
+            wrapt.wrap_function_wrapper(
+                "aworld.core.agent.base",
+                "BaseAgent.async_run",
+                _async_run_class_wrapper(tracer=tracer)
+            )
+            wrapt.wrap_function_wrapper(
+                "aworld.agents.llm_agent",
+                "Agent.invoke_model",
+                _call_llm_model_class_wrapper(tracer=tracer)
+            )
+        except Exception as err:
+            logger.warning(f"AgentInstrumentor#_instrument failed ,err is {err}")
+
+    def _uninstrument(self, **kwargs: Any):
+        pass
+
+
+def wrap_agent(agent: 'aworld.core.agent.base.BaseAgent'):
+    try:
+        tracer_provider = get_tracer_provider_silent()
+        if not tracer_provider:
+            return agent
+        tracer = tracer_provider.get_tracer(
+            "aworld.trace.instrumentation.agent")
+
+        async_run_wrapper = _async_run_instance_wrapper(tracer)
+        agent.async_run = async_run_wrapper(agent.async_run)
+        if hasattr(agent, "_call_llm_model"):
+            call_llm_model_wrapper = _call_llm_model_instance_wrapper(tracer)
+            agent._call_llm_model = call_llm_model_wrapper(agent._call_llm_model)
+    except Exception:
+        logger.warning(traceback.format_exc())
+
+    return agent

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/asgi.py

@@ -0,0 +1,203 @@
+from timeit import default_timer
+from typing import Any, Awaitable, Callable
+from functools import wraps
+from aworld.metrics.context_manager import MetricContext
+from aworld.trace.instrumentation.http_util import (
+    collect_request_attributes_asgi,
+    url_disabled,
+    parser_host_port_url_from_asgi
+)
+from aworld.trace.base import Span, TraceProvider, TraceContext, Tracer, SpanType
+from aworld.trace.propagator import get_global_trace_propagator
+from aworld.trace.propagator.carrier import DictCarrier, ListTupleCarrier
+from aworld.metrics.metric import MetricType
+from aworld.metrics.template import MetricTemplate
+from aworld.logs.util import logger
+
+
+def _wrapped_receive(
+    server_span: Span,
+    server_span_name: str,
+    scope: dict[str, Any],
+    receive: Callable[[], Awaitable[dict[str, Any]]],
+    attributes: dict[str],
+    client_request_hook: Callable = None
+):
+
+    @wraps(receive)
+    async def otel_receive():
+        message = await receive()
+        if client_request_hook and callable(client_request_hook):
+            client_request_hook(scope, message)
+
+        server_span.set_attribute("asgi.event.type", message.get("type", ""))
+        return message
+
+    return otel_receive
+
+
+def _wrapped_send(
+    server_span: Span,
+    server_span_name: str,
+    scope: dict[str, Any],
+    send: Callable[[dict[str, Any]], Awaitable[None]],
+    attributes: dict[str],
+    client_response_hook: Callable = None
+):
+    expecting_trailers = False
+
+    @wraps(send)
+    async def otel_send(message: dict[str, Any]):
+        nonlocal expecting_trailers
+
+        status_code = None
+        if message["type"] == "http.response.start":
+            status_code = message["status"]
+        elif message["type"] == "websocket.send":
+            status_code = 200
+
+        # raw_headers = message.get("headers")
+        # if raw_headers:
+        if status_code:
+            server_span.set_attribute(
+                "http.response.status_code", status_code)
+
+        if callable(client_response_hook):
+            client_response_hook(scope, message)
+
+        if message["type"] == "http.response.start":
+            expecting_trailers = message.get("trailers", False)
+
+        propagator = get_global_trace_propagator()
+        if propagator:
+            trace_context = TraceContext(
+                trace_id=server_span.get_trace_id(),
+                span_id=server_span.get_span_id()
+            )
+            propagator.inject(
+                trace_context, DictCarrier(message))
+
+        await send(message)
+
+        if (
+            not expecting_trailers
+            and message["type"] == "http.response.body"
+            and not message.get("more_body", False)
+        ) or (
+            expecting_trailers
+            and message["type"] == "http.response.trailers"
+            and not message.get("more_trailers", False)
+        ):
+            server_span.end()
+
+    return otel_send
+
+
+class TraceMiddleware:
+    """
+    A ASGI Middleware for tracing requests and responses.
+    """
+
+    def __init__(
+            self,
+            app,
+            excluded_urls=None,
+            tracer_provider: TraceProvider = None,
+            tracer: Tracer = None,
+            server_request_hook: Callable = None,
+            client_request_hook: Callable = None,
+            client_response_hook: Callable = None,):
+        self.app = app
+        self.excluded_urls = excluded_urls
+        self.tracer_provider = tracer_provider
+        self.server_request_hook = server_request_hook
+        self.client_request_hook = client_request_hook
+        self.client_response_hook = client_response_hook
+
+        self.tracer: Tracer = (self.tracer_provider.get_tracer(
+            "aworld.trace.instrumentation.asgi"
+        ) if tracer is None else tracer)
+
+        self.duration_histogram = MetricTemplate(
+            type=MetricType.HISTOGRAM,
+            name="asgi_request_duration_histogram",
+            description="Duration of flask HTTP server requests."
+        )
+
+        self.active_requests_counter = MetricTemplate(
+            type=MetricType.UPDOWNCOUNTER,
+            name="asgi_active_request_counter",
+            unit="1",
+            description="Number of active HTTP server requests.",
+        )
+
+    async def __call__(
+        self,
+        scope: dict[str, Any],
+        receive: Callable[[], Awaitable[dict[str, Any]]],
+        send: Callable[[dict[str, Any]], Awaitable[None]],
+    ):
+        start = default_timer()
+        if scope["type"] not in ("http", "websocket"):
+            return await self.app(scope, receive, send)
+
+        _, _, url = parser_host_port_url_from_asgi(scope)
+        if self.excluded_urls and url_disabled(url, self.excluded_urls):
+            return await self.app(scope, receive, send)
+
+        span_name = scope.get("method", "HTTP").strip(
+        ).upper() + "_" + scope.get("path", "").strip()
+
+        attributes = collect_request_attributes_asgi(scope)
+
+        if scope["type"] == "http" and MetricContext.metric_initialized():
+            MetricContext.inc(self.active_requests_counter, 1, attributes)
+
+        trace_context = None
+        propagator = get_global_trace_propagator()
+        if propagator:
+            trace_context = propagator.extract(
+                ListTupleCarrier(scope.get("headers", [])))
+            logger.info(
+                f"asgi extract trace_context: {trace_context}, scope: {scope}")
+        try:
+            with self.tracer.start_as_current_span(
+                span_name, span_type=SpanType.SERVER, trace_context=trace_context, attributes=attributes
+            ) as span:
+
+                if callable(self.server_request_hook):
+                    self.server_request_hook(scope)
+
+                wrappered_receive = _wrapped_receive(
+                    span,
+                    span_name,
+                    scope,
+                    receive,
+                    attributes,
+                    self.client_request_hook
+                )
+                wrappered_send = _wrapped_send(
+                    span,
+                    span_name,
+                    scope,
+                    send,
+                    attributes,
+                    self.client_response_hook
+                )
+
+                await self.app(scope, wrappered_receive, wrappered_send)
+        finally:
+            if scope["type"] == "http":
+                duration_s = default_timer() - start
+
+                if MetricContext.metric_initialized():
+                    MetricContext.histogram_record(
+                        self.duration_histogram,
+                        duration_s,
+                        attributes
+                    )
+                    MetricContext.inc(
+                        self.active_requests_counter, -1, attributes)
+
+            if span.is_recording():
+                span.end()

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/eventbus.py

@@ -0,0 +1,115 @@
+import wrapt
+from typing import Any, Collection
+from aworld.trace.instrumentation import Instrumentor
+from aworld.trace.base import Tracer, get_tracer_provider_silent, TraceContext
+from aworld.trace.propagator import get_global_trace_propagator, get_global_trace_context
+from aworld.trace.propagator.carrier import DictCarrier
+from aworld.logs.util import logger
+
+
+def _emit_message_class_wrapper(tracer: Tracer):
+    async def awrapper(wrapped, instance, args, kwargs):
+        from aworld.core.event.base import Message
+        try:
+            event = args[0] if len(args) > 0 else kwargs.get("event")
+            propagator = get_global_trace_propagator()
+            trace_provider = get_tracer_provider_silent()
+            if trace_provider and propagator and event and isinstance(event, Message):
+                if not event.headers:
+                    event.headers = {}
+                current_span = trace_provider.get_current_span()
+                if current_span:
+                    trace_context = TraceContext(
+                        trace_id=current_span.get_trace_id(), span_id=current_span.get_span_id())
+                    propagator.inject(trace_context=trace_context,
+                                      carrier=DictCarrier(event.headers))
+                logger.info(
+                    f"EventManager emit_message trace propagate, event.headers={event.headers}")
+        except Exception as e:
+            logger.error(
+                f"EventManager emit_message trace propagate exception: {e}")
+        return await wrapped(*args, **kwargs)
+    return awrapper
+
+
+def _emit_message_instance_wrapper(tracer: Tracer):
+
+    @wrapt.decorator
+    async def awrapper(wrapped, instance, args, kwargs):
+        wrapper = _emit_message_class_wrapper(tracer)
+        return await wrapper(wrapped, instance, args, kwargs)
+
+    return awrapper
+
+
+def _consume_class_wrapper(tracer: Tracer):
+    async def awrapper(wrapped, instance, args, kwargs):
+        from aworld.core.event.base import Message
+        event = await wrapped(*args, **kwargs)
+        try:
+            propagator = get_global_trace_propagator()
+            if propagator and event and isinstance(event, Message) and event.headers:
+                trace_context = propagator.extract(DictCarrier(event.headers))
+                # logger.info(
+                #     f"extract trace_context from event: {trace_context}")
+                if trace_context:
+                    get_global_trace_context().set(trace_context)
+        except Exception as e:
+            logger.error(
+                f"EventManager consume trace propagate exception: {e}")
+        return event
+    return awrapper
+
+
+def _consume_instance_wrapper(tracer: Tracer):
+
+    @wrapt.decorator
+    async def awrapper(wrapped, instance, args, kwargs):
+        wrapper = _consume_class_wrapper(tracer)
+        return await wrapper(wrapped, instance, args, kwargs)
+
+    return awrapper
+
+
+class EventBusInstrumentor(Instrumentor):
+
+    def instrumentation_dependencies(self) -> Collection[str]:
+        return ()
+
+    def _uninstrument(self, **kwargs: Any):
+        pass
+
+    def _instrument(self, **kwargs: Any):
+        tracer_provider = get_tracer_provider_silent()
+        if not tracer_provider:
+            return
+        tracer = tracer_provider.get_tracer(
+            "aworld.trace.instrumentation.eventbus")
+
+        wrapt.wrap_function_wrapper(
+            "aworld.events.manager",
+            "EventManager.emit_message",
+            _emit_message_class_wrapper(tracer=tracer)
+        )
+
+        wrapt.wrap_function_wrapper(
+            "aworld.events.manager",
+            "EventManager.consume",
+            _consume_class_wrapper(tracer=tracer)
+        )
+
+
+def wrap_event_manager(manager: 'aworld.events.manager.EventManager'):
+    tracer_provider = get_tracer_provider_silent()
+    if not tracer_provider:
+        return manager
+    tracer = tracer_provider.get_tracer(
+        "aworld.trace.instrumentation.eventbus")
+
+    emit_wrapper = _emit_message_instance_wrapper(tracer)
+    consume_wrapper = _consume_instance_wrapper(tracer)
+
+    manager.emit_message = emit_wrapper(manager.emit_message)
+    manager.consume = consume_wrapper(manager.consume)
+
+    return manager

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/fastapi.py

@@ -0,0 +1,107 @@
+from typing import Any, Callable
+from .asgi import TraceMiddleware
+from aworld.trace.instrumentation import Instrumentor
+from aworld.trace.base import TraceProvider, get_tracer_provider
+from aworld.trace.instrumentation.http_util import (
+    get_excluded_urls,
+    parse_excluded_urls,
+)
+from aworld.utils.import_package import import_packages
+from aworld.logs.util import logger
+
+import_packages(['fastapi'])  # noqa
+import fastapi  # noqa
+
+
+class _InstrumentedFastAPI(fastapi.FastAPI):
+    """Instrumented FastAPI class."""
+    _tracer_provider: TraceProvider = None
+    _excluded_urls: list[str] = None
+    _server_request_hook: Callable = None
+    _client_request_hook: Callable = None
+    _client_response_hook: Callable = None
+    _instrumented_fastapi_apps = set()
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        tracer = self._tracer_provider.get_tracer(
+            "aworld.trace.instrumentation.fastapi")
+
+        self.add_middleware(
+            TraceMiddleware,
+            tracer=tracer,
+            excluded_urls=self._excluded_urls,
+            server_request_hook=self._server_request_hook,
+            client_request_hook=self._client_request_hook,
+            client_response_hook=self._client_response_hook
+        )
+
+        self._is_instrumented_by_trace = True
+        self._instrumented_fastapi_apps.add(self)
+
+    def __del__(self):
+        if self in self._instrumented_fastapi_apps:
+            self._instrumented_fastapi_apps.remove(self)
+
+
+class FastAPIInstrumentor(Instrumentor):
+    """FastAPI Instrumentor."""
+    _original_fastapi = None
+
+    @staticmethod
+    def uninstrument_app(app: fastapi.FastAPI):
+        app.user_middleware = [
+            x
+            for x in app.user_middleware
+            if x.cls is not TraceMiddleware
+        ]
+        app.middleware_stack = app.build_middleware_stack()
+        app._is_instrumented_by_trace = False
+
+    def instrumentation_dependencies(self) -> dict[str, Any]:
+        return {"fastapi": fastapi}
+
+    def _instrument(self, **kwargs):
+        self._original_fastapi = fastapi.FastAPI
+        _InstrumentedFastAPI._tracer_provider = kwargs.get("tracer_provider")
+        _InstrumentedFastAPI._server_request_hook = kwargs.get(
+            "server_request_hook"
+        )
+        _InstrumentedFastAPI._client_request_hook = kwargs.get(
+            "client_request_hook"
+        )
+        _InstrumentedFastAPI._client_response_hook = kwargs.get(
+            "client_response_hook"
+        )
+        excluded_urls = kwargs.get("excluded_urls")
+        _InstrumentedFastAPI._excluded_urls = (
+            get_excluded_urls("FASTAPI")
+            if excluded_urls is None
+            else parse_excluded_urls(excluded_urls)
+        )
+        fastapi.FastAPI = _InstrumentedFastAPI
+
+    def _uninstrument(self, **kwargs):
+        for app in _InstrumentedFastAPI._instrumented_fastapi_apps:
+            self.uninstrument_app(app)
+        _InstrumentedFastAPI._instrumented_fastapi_apps.clear()
+        fastapi.FastAPI = self._original_fastapi
+
+
+def instrument_fastapi(excluded_urls: str = None,
+                       server_request_hook: Callable = None,
+                       client_request_hook: Callable = None,
+                       client_response_hook: Callable = None,
+                       tracer_provider: TraceProvider = None,
+                       **kwargs: Any,
+                       ):
+    kwargs.update({
+        "excluded_urls": excluded_urls,
+        "server_request_hook": server_request_hook,
+        "client_request_hook": client_request_hook,
+        "client_response_hook": client_response_hook,
+        "tracer_provider": tracer_provider or get_tracer_provider(),
+    })
+    FastAPIInstrumentor().instrument(**kwargs)
+    logger.info("FastAPI instrumented.")

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/flask.py

@@ -0,0 +1,279 @@
+import flask
+import weakref
+from typing import Any, Callable, Collection
+from time import time_ns
+from timeit import default_timer
+from importlib_metadata import version
+from packaging import version as package_version
+from aworld.trace.instrumentation import Instrumentor
+from aworld.trace.base import Span, TraceProvider, TraceContext, Tracer, SpanType, get_tracer_provider
+from aworld.metrics.metric import MetricType
+from aworld.metrics.template import MetricTemplate
+from aworld.logs.util import logger
+from aworld.trace.instrumentation.http_util import (
+    collect_request_attributes,
+    url_disabled,
+    get_excluded_urls,
+    parse_excluded_urls,
+    HTTP_ROUTE
+)
+from aworld.trace.propagator import get_global_trace_propagator
+from aworld.metrics.context_manager import MetricContext
+from aworld.trace.propagator.carrier import ListTupleCarrier, DictCarrier
+
+_ENVIRON_STARTTIME_KEY = "aworld-flask.starttime_key"
+_ENVIRON_SPAN_KEY = "aworld-flask.span_key"
+_ENVIRON_REQCTX_REF_KEY = "aworld-flask.reqctx_ref_key"
+
+flask_version = version("flask")
+if package_version.parse(flask_version) >= package_version.parse("2.2.0"):
+
+    def _request_ctx_ref() -> weakref.ReferenceType:
+        return weakref.ref(flask.globals.request_ctx._get_current_object())
+
+else:
+
+    def _request_ctx_ref() -> weakref.ReferenceType:
+        return weakref.ref(flask._request_ctx_stack.top)
+
+
+def _rewrapped_app(
+    wsgi_app,
+    active_requests_counter,
+    duration_histogram,
+    response_hook=None,
+    excluded_urls=None,
+):
+    def _wrapped_app(wrapped_app_environ, start_response):
+        # We want to measure the time for route matching, etc.
+        # In theory, we could start the span here and use
+        # update_name later but that API is "highly discouraged" so
+        # we better avoid it.
+        wrapped_app_environ[_ENVIRON_STARTTIME_KEY] = time_ns()
+        start = default_timer()
+        attributes = collect_request_attributes(wrapped_app_environ)
+
+        if MetricContext.metric_initialized():
+            MetricContext.inc(active_requests_counter, 1, attributes)
+
+        request_route = None
+
+        def _start_response(status, response_headers, *args, **kwargs):
+            if flask.request and (
+                excluded_urls is None
+                or not url_disabled(flask.request.url, excluded_urls)
+            ):
+                nonlocal request_route
+                request_route = flask.request.url_rule
+
+                span: Span = flask.request.environ.get(_ENVIRON_SPAN_KEY)
+
+                propagator = get_global_trace_propagator()
+                if propagator and span:
+                    trace_context = TraceContext(
+                        trace_id=span.get_trace_id(),
+                        span_id=span.get_span_id()
+                    )
+                    propagator.inject(
+                        trace_context, ListTupleCarrier(response_headers))
+
+                if span and span.is_recording():
+                    status_code_str, _ = status.split(" ", 1)
+                    try:
+                        status_code = int(status_code_str)
+                    except ValueError:
+                        status_code = -1
+
+                    span.set_attribute(
+                        "http.response.status_code", status_code)
+                    span.set_attributes(attributes)
+
+                if response_hook is not None:
+                    response_hook(span, status, response_headers)
+            return start_response(status, response_headers, *args, **kwargs)
+
+        result = wsgi_app(wrapped_app_environ, _start_response)
+        duration_s = default_timer() - start
+
+        if MetricContext.metric_initialized():
+            MetricContext.histogram_record(
+                duration_histogram,
+                duration_s,
+                attributes
+            )
+            MetricContext.dec(active_requests_counter, 1, attributes)
+        return result
+
+    return _wrapped_app
+
+
+def _wrapped_before_request(
+    request_hook=None,
+    tracer: Tracer = None,
+    excluded_urls=None
+):
+    def _before_request():
+        if excluded_urls and url_disabled(flask.request.url, excluded_urls):
+            return
+        flask_request_environ = flask.request.environ
+        logger.info(
+            f"_wrapped_before_request flask_request_environ={flask_request_environ}")
+
+        attributes = collect_request_attributes(flask_request_environ)
+
+        if flask.request.url_rule:
+            # For 404 that result from no route found, etc, we
+            # don't have a url_rule.
+            attributes[HTTP_ROUTE] = flask.request.url_rule.rule
+            span_name = f"HTTP {flask.request.url_rule.rule}"
+        else:
+            span_name = f"HTTP {flask.request.url}"
+
+        propagator = get_global_trace_propagator()
+        trace_context = None
+        if propagator:
+            trace_context = propagator.extract(
+                DictCarrier(flask_request_environ))
+
+        logger.info(f"_wrapped_before_request trace_context={trace_context}")
+
+        span = tracer.start_span(
+            span_name,
+            SpanType.SERVER,
+            attributes=attributes,
+            start_time=flask_request_environ.get(_ENVIRON_STARTTIME_KEY),
+            trace_context=trace_context
+        )
+
+        if request_hook:
+            request_hook(span, flask_request_environ)
+
+        flask_request_environ[_ENVIRON_SPAN_KEY] = span
+        flask_request_environ[_ENVIRON_REQCTX_REF_KEY] = _request_ctx_ref()
+
+    return _before_request
+
+
+def _wrapped_teardown_request(
+    excluded_urls=None,
+):
+    def _teardown_request(exc):
+        if excluded_urls and url_disabled(flask.request.url, excluded_urls):
+            return
+
+        span: Span = flask.request.environ.get(_ENVIRON_SPAN_KEY)
+
+        original_reqctx_ref = flask.request.environ.get(
+            _ENVIRON_REQCTX_REF_KEY
+        )
+        current_reqctx_ref = _request_ctx_ref()
+        if not span or original_reqctx_ref != current_reqctx_ref:
+            return
+        if exc is None:
+            span.end()
+        else:
+            span.record_exception(exc)
+            span.end()
+
+    return _teardown_request
+
+
+class _InstrumentedFlask(flask.Flask):
+    _excluded_urls = None
+    _tracer_provider: TraceProvider = None
+    _request_hook = None
+    _response_hook = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        tracer = self._tracer_provider.get_tracer(
+            "aworld.trace.instrumentation.flask")
+
+        duration_histogram = MetricTemplate(
+            type=MetricType.HISTOGRAM,
+            name="flask_request_duration_histogram",
+            description="Duration of flask HTTP server requests."
+        )
+
+        active_requests_counter = MetricTemplate(
+            type=MetricType.UPDOWNCOUNTER,
+            name="flask_active_request_counter",
+            unit="1",
+            description="Number of active HTTP server requests.",
+        )
+
+        self.wsgi_app = _rewrapped_app(
+            self.wsgi_app,
+            active_requests_counter,
+            duration_histogram,
+            _InstrumentedFlask._response_hook,
+            excluded_urls=_InstrumentedFlask._excluded_urls
+        )
+
+        _before_request = _wrapped_before_request(
+            _InstrumentedFlask._request_hook,
+            tracer,
+            excluded_urls=_InstrumentedFlask._excluded_urls
+        )
+        self._before_request = _before_request
+        self.before_request(_before_request)
+
+        _teardown_request = _wrapped_teardown_request(
+            excluded_urls=_InstrumentedFlask._excluded_urls,
+        )
+        self.teardown_request(_teardown_request)
+
+
+class FlaskInstrumentor(Instrumentor):
+
+    def instrumentation_dependencies(self) -> Collection[str]:
+        return ("flask >= 1.0",)
+
+    def _instrument(self, **kwargs: Any):
+        logger.info("Flask _instrument entered.")
+        self._original_flask = flask.Flask
+        request_hook = kwargs.get("request_hook")
+        response_hook = kwargs.get("response_hook")
+        if callable(request_hook):
+            _InstrumentedFlask._request_hook = request_hook
+        if callable(response_hook):
+            _InstrumentedFlask._response_hook = response_hook
+        tracer_provider = kwargs.get("tracer_provider")
+        _InstrumentedFlask._tracer_provider = tracer_provider
+        excluded_urls = kwargs.get("excluded_urls")
+        _InstrumentedFlask._excluded_urls = (
+            get_excluded_urls("FLASK")
+            if excluded_urls is None
+            else parse_excluded_urls(excluded_urls)
+        )
+        flask.Flask = _InstrumentedFlask
+        logger.info("Flask _instrument exited.")
+
+    def _uninstrument(self, **kwargs):
+        flask.Flask = self._original_flask
+
+
+def instrument_flask(excluded_urls: str = None,
+                     request_hook: Callable = None,
+                     response_hook: Callable = None,
+                     tracer_provider: TraceProvider = None,
+                     **kwargs: Any,
+                     ):
+    """
+    Instrument the Flask application.
+    Args:
+        excluded_urls (str): A comma separated list of URLs to be excluded from instrumentation.
+        request_hook (Callable): A function to be called before a request is processed.
+        response_hook (Callable): A function to be called after a request is processed.
+        tracer_provider (TraceProvider): The trace provider to use.
+    """
+    all_kwargs = {
+        "excluded_urls": excluded_urls,
+        "request_hook": request_hook,
+        "response_hook": response_hook,
+        "tracer_provider": tracer_provider or get_tracer_provider(),
+        **kwargs
+    }
+    FlaskInstrumentor().instrument(**all_kwargs)
+    logger.info("Flask instrumented.")

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/http_util.py

@@ -0,0 +1,193 @@
+import os
+from re import compile as re_compile
+from re import search
+from typing import Final, Iterable, Any
+from urllib.parse import urlparse, urlunparse, unquote
+from wsgiref.types import WSGIEnvironment
+from requests.models import PreparedRequest
+
+HTTP_REQUEST_METHOD: Final = "http.request.method"
+HTTP_FLAVOR: Final = "http.flavor"
+HTTP_HOST: Final = "http.host"
+HTTP_SCHEME: Final = "http.scheme"
+HTTP_USER_AGENT: Final = "http.user_agent"
+HTTP_SERVER_NAME: Final = "http.server_name"
+SERVER_ADDRESS: Final = "server.address"
+SERVER_PORT: Final = "server.port"
+URL_PATH: Final = "url.path"
+URL_QUERY: Final = "url.query"
+CLIENT_ADDRESS: Final = "client.address"
+CLIENT_PORT: Final = "client.port"
+URL_FULL: Final = "url.full"
+
+HTTP_REQUEST_BODY_SIZE: Final = "http.request.body.size"
+HTTP_REQUEST_HEADER: Final = "http.request.header"
+HTTP_REQUEST_SIZE: Final = "http.request.size"
+HTTP_RESPONSE_BODY_SIZE: Final = "http.response.body.size"
+HTTP_RESPONSE_HEADER: Final = "http.response.header"
+HTTP_RESPONSE_SIZE: Final = "http.response.size"
+HTTP_RESPONSE_STATUS_CODE: Final = "http.response.status_code"
+HTTP_ROUTE = "http.route"
+
+
+def collect_request_attributes(environ: WSGIEnvironment):
+
+    attributes: dict[str] = {}
+
+    request_method = environ.get("REQUEST_METHOD", "")
+    request_method = request_method.upper()
+    attributes[HTTP_REQUEST_METHOD] = request_method
+    attributes[HTTP_FLAVOR] = environ.get("SERVER_PROTOCOL", "")
+    attributes[HTTP_SCHEME] = environ.get("wsgi.url_scheme", "")
+    attributes[HTTP_SERVER_NAME] = environ.get("SERVER_NAME", "")
+    attributes[HTTP_HOST] = environ.get("HTTP_HOST", "")
+    host_port = environ.get("SERVER_PORT")
+    if host_port:
+        attributes[SERVER_PORT] = host_port
+    target = environ.get("RAW_URI")
+    if target is None:
+        target = environ.get("REQUEST_URI")
+    if target:
+        path, query = _parse_url_query(target)
+        attributes[URL_PATH] = path
+        attributes[URL_QUERY] = query
+    remote_addr = environ.get("REMOTE_ADDR", "")
+    attributes[CLIENT_ADDRESS] = remote_addr
+    attributes[CLIENT_PORT] = environ.get("REMOTE_PORT", "")
+    remote_host = environ.get("REMOTE_HOST")
+    if remote_host and remote_host != remote_addr:
+        attributes[CLIENT_ADDRESS] = remote_host
+    attributes[HTTP_USER_AGENT] = environ.get("HTTP_USER_AGENT", "")
+    return attributes
+
+
+def collect_attributes_from_request(request: PreparedRequest) -> dict[str]:
+    attributes: dict[str] = {}
+
+    url = remove_url_credentials(request.url)
+    attributes[HTTP_REQUEST_METHOD] = request.method
+    attributes[URL_FULL] = url
+    parsed_url = urlparse(url)
+    if parsed_url.scheme:
+        attributes[HTTP_SCHEME] = parsed_url.scheme
+    if parsed_url.hostname:
+        attributes[HTTP_HOST] = parsed_url.hostname
+    if parsed_url.port:
+        attributes[SERVER_PORT] = parsed_url.port
+    return attributes
+
+
+def url_disabled(url: str, excluded_urls: Iterable[str]) -> bool:
+    """
+    Check if the url is disabled.
+    Args:
+        url: The url to check.
+        excluded_urls: The excluded urls.
+    Returns:
+        True if the url is disabled, False otherwise.
+    """
+    if excluded_urls is None:
+        return False
+    regex = re_compile("|".join(excluded_urls))
+    return search(regex, url)
+
+
+def get_excluded_urls(instrumentation: str) -> list[str]:
+    """
+    Get the excluded urls.
+    Args:
+        instrumentation: The instrumentation to get the excluded urls for.
+    Returns:
+        The excluded urls.
+    """
+
+    excluded_urls = os.environ.get(f"{instrumentation}_EXCLUDED_URLS")
+
+    return parse_excluded_urls(excluded_urls)
+
+
+def parse_excluded_urls(excluded_urls: str) -> list[str]:
+    """
+    Parse the excluded urls.
+    Args:
+        excluded_urls: The excluded urls.
+    Returns:
+        The excluded urls.
+    """
+    if excluded_urls:
+        excluded_url_list = [
+            excluded_url.strip() for excluded_url in excluded_urls.split(",")
+        ]
+    else:
+        excluded_url_list = []
+
+    return excluded_url_list
+
+
+def remove_url_credentials(url: str) -> str:
+    """Given a string url, remove the username and password only if it is a valid url"""
+
+    try:
+        parsed = urlparse(url)
+        if all([parsed.scheme, parsed.netloc]):  # checks for valid url
+            parsed_url = urlparse(url)
+            _, _, netloc = parsed.netloc.rpartition("@")
+            return urlunparse(
+                (
+                    parsed_url.scheme,
+                    netloc,
+                    parsed_url.path,
+                    parsed_url.params,
+                    parsed_url.query,
+                    parsed_url.fragment,
+                )
+            )
+    except ValueError:  # an unparsable url was passed
+        pass
+    return url
+
+
+def parser_host_port_url_from_asgi(scope: dict[str, Any]):
+    """Returns (host, port, full_url) tuple."""
+    server = scope.get("server") or ["0.0.0.0", 80]
+    port = server[1]
+    server_host = server[0] + (":" + str(port) if str(port) != "80" else "")
+    full_path = scope.get("path", "")
+    http_url = scope.get("scheme", "http") + "://" + server_host + full_path
+    return server_host, port, http_url
+
+
+def collect_request_attributes_asgi(scope: dict[str, Any]):
+    attributes: dict[str] = {}
+    server_host, port, http_url = parser_host_port_url_from_asgi(scope)
+    query_string = scope.get("query_string")
+    if query_string and http_url:
+        if isinstance(query_string, bytes):
+            query_string = query_string.decode("utf8")
+        http_url += "?" + unquote(query_string)
+    attributes[HTTP_REQUEST_METHOD] = scope.get("method", "")
+    attributes[HTTP_FLAVOR] = scope.get("http_version", "")
+    attributes[HTTP_SCHEME] = scope.get("scheme", "")
+    attributes[HTTP_HOST] = server_host
+    attributes[SERVER_PORT] = port
+    attributes[URL_FULL] = remove_url_credentials(http_url)
+    attributes[URL_PATH] = scope.get("path", "")
+    header = scope.get("headers")
+    if header:
+        for key, value in header:
+            if key == b"user-agent":
+                attributes[HTTP_USER_AGENT] = value.decode("utf8")
+
+    client = scope.get("client")
+    if client:
+        attributes[CLIENT_ADDRESS] = client[0]
+        attributes[CLIENT_PORT] = client[1]
+
+    return attributes
+
+
+def _parse_url_query(url: str):
+    parsed_url = urlparse(url)
+    path = parsed_url.path
+    query_params = parsed_url.query
+    return path, query_params

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/llm_metrics.py

@@ -0,0 +1,118 @@
+from aworld.metrics.context_manager import MetricContext
+from aworld.metrics.template import MetricTemplate
+from aworld.metrics.metric import MetricType
+
+tokens_usage_histogram = MetricTemplate(
+    type=MetricType.HISTOGRAM,
+    name="llm_token_usage",
+    unit="token",
+    description="Measures number of input and output tokens used"
+)
+
+chat_choice_counter = MetricTemplate(
+    type=MetricType.COUNTER,
+    name="llm_generation_choice_counter",
+    unit="choice",
+    description="Number of choices returned by chat completions call"
+)
+
+duration_histogram = MetricTemplate(
+    type=MetricType.HISTOGRAM,
+    name="llm_chat_duration",
+    unit="s",
+    description="AI chat duration",
+)
+
+chat_exception_counter = MetricTemplate(
+    type=MetricType.COUNTER,
+    name="llm_chat_exception_counter",
+    unit="time",
+    description="Number of exceptions occurred during chat completions",
+)
+
+streaming_time_to_first_token_histogram = MetricTemplate(
+    type=MetricType.HISTOGRAM,
+    name="llm_streaming_time_to_first_token",
+    unit="s",
+    description="Time to first token in streaming chat completions",
+)
+streaming_time_to_generate_histogram = MetricTemplate(
+    type=MetricType.HISTOGRAM,
+    name="streaming_time_to_generate",
+    unit="s",
+    description="Time between first token and completion in streaming chat completions",
+)
+
+
+def record_exception_metric(exception, duration):
+    '''
+    record chat exception to metrics
+    '''
+    if MetricContext.metric_initialized():
+        labels = {
+            "error.type": exception.__class__.__name__,
+        }
+        if duration_histogram:
+            MetricContext.histogram_record(
+                duration_histogram, duration, labels=labels)
+        if chat_exception_counter:
+            MetricContext.count(
+                chat_exception_counter, 1, labels=labels)
+
+
+def record_streaming_time_to_first_token(duration, labels):
+    '''
+    Record duration of start time to first token in stream. 
+    '''
+    if MetricContext.metric_initialized():
+        MetricContext.histogram_record(
+            streaming_time_to_first_token_histogram, duration, labels=labels)
+
+
+def record_streaming_time_to_generate(first_token_to_generate_duration, labels):
+    '''
+    Record duration the first token to response to generation
+    '''
+    if MetricContext.metric_initialized():
+        MetricContext.histogram_record(
+            streaming_time_to_generate_histogram, first_token_to_generate_duration, labels=labels)
+
+
+def record_chat_response_metric(attributes,
+                                prompt_tokens,
+                                completion_tokens,
+                                duration,
+                                choices=None
+                                ):
+    '''
+    Record chat response to metrics
+    '''
+    if MetricContext.metric_initialized():
+        if prompt_tokens and tokens_usage_histogram:
+            labels = {
+                **attributes,
+                "llm.prompt_usage_type": "prompt_tokens"
+            }
+            MetricContext.histogram_record(
+                tokens_usage_histogram, prompt_tokens, labels=labels)
+        if completion_tokens and tokens_usage_histogram:
+            labels = {
+                **attributes,
+                "llm.prompt_usage_type": "completion_tokens"
+            }
+            MetricContext.histogram_record(
+                tokens_usage_histogram, completion_tokens, labels=labels)
+        if duration and duration_histogram:
+            MetricContext.histogram_record(
+                duration_histogram, duration, labels=attributes)
+        if choices and chat_choice_counter:
+            MetricContext.count(chat_choice_counter,
+                                len(choices), labels=attributes)
+            for choice in choices:
+                if choice.get("finish_reason"):
+                    finish_reason_attr = {
+                        **attributes,
+                        "llm.finish_reason": choice.get("finish_reason")
+                    }
+                    MetricContext.count(
+                        chat_choice_counter, 1, labels=finish_reason_attr)

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/openai/__init__.py

@@ -0,0 +1,333 @@
+import wrapt
+import time
+import openai
+import traceback
+import aworld.trace.instrumentation.semconv as semconv
+from typing import Collection, Any, Union
+from aworld.trace.instrumentation import Instrumentor
+from aworld.trace.base import (
+    Tracer,
+    SpanType,
+    get_tracer_provider_silent
+)
+from aworld.trace.constants import ATTRIBUTES_MESSAGE_RUN_TYPE_KEY, RunType
+from aworld.trace.instrumentation.openai.inout_parse import (
+    run_async,
+    handle_openai_request,
+    is_streaming_response,
+    record_stream_response_chunk,
+    parse_openai_response,
+    record_stream_token_usage,
+    model_as_dict,
+    parse_response_message,
+)
+from aworld.trace.instrumentation.llm_metrics import (
+    record_exception_metric,
+    record_chat_response_metric,
+    record_streaming_time_to_first_token,
+    record_streaming_time_to_generate
+)
+from aworld.logs.util import logger
+
+
+def _chat_wrapper(tracer: Tracer):
+
+    @wrapt.decorator
+    def wrapper(wrapped, instance, args, kwargs):
+        model_name = kwargs.get("model", "")
+        if not model_name:
+            model_name = "OpenAI"
+        span_attributes = {}
+        span_attributes[ATTRIBUTES_MESSAGE_RUN_TYPE_KEY] = RunType.LLM.value
+
+        span = tracer.start_span(
+            name=model_name, span_type=SpanType.CLIENT, attributes=span_attributes)
+
+        run_async(handle_openai_request(span, kwargs, instance))
+        start_time = time.time()
+        try:
+            response = wrapped(*args, **kwargs)
+        except Exception as e:
+            record_exception(span=span,
+                             start_time=start_time,
+                             exception=e
+                             )
+            span.end()
+            raise e
+
+        if is_streaming_response(response):
+            return WrappedStreamResponse(span=span,
+                                         response=response,
+                                         instance=instance,
+                                         start_time=start_time,
+                                         request_kwargs=kwargs
+                                         )
+
+        record_completion(span=span,
+                          start_time=start_time,
+                          response=response,
+                          request_kwargs=kwargs,
+                          instance=instance
+                          )
+        span.end()
+        return response
+
+    return wrapper
+
+
+def _achat_class_wrapper(tracer: Tracer):
+
+    async def awrapper(wrapped, instance, args, kwargs):
+        model_name = kwargs.get("model", "")
+        if not model_name:
+            model_name = "OpenAI"
+        span_attributes = {}
+        span_attributes[ATTRIBUTES_MESSAGE_RUN_TYPE_KEY] = RunType.LLM.value
+
+        span = tracer.start_span(
+            name=model_name, span_type=SpanType.CLIENT, attributes=span_attributes)
+
+        await handle_openai_request(span, kwargs, instance)
+        start_time = time.time()
+        try:
+            response = await wrapped(*args, **kwargs)
+        except Exception as e:
+            record_exception(span=span,
+                             start_time=start_time,
+                             exception=e
+                             )
+            span.end()
+            raise e
+
+        if is_streaming_response(response):
+            return WrappedStreamResponse(span=span,
+                                         response=response,
+                                         instance=instance,
+                                         start_time=start_time,
+                                         request_kwargs=kwargs
+                                         )
+        record_completion(span=span,
+                          start_time=start_time,
+                          response=response,
+                          request_kwargs=kwargs,
+                          instance=instance
+                          )
+        span.end()
+        return response
+
+    return awrapper
+
+
+def _achat_instance_wrapper(tracer: Tracer):
+
+    @wrapt.decorator
+    async def _awrapper(wrapped, instance, args, kwargs):
+        wrapper_func = _achat_class_wrapper(tracer)
+        return await wrapper_func(wrapped, instance, args, kwargs)
+
+    return _awrapper
+
+
+def record_exception(span, start_time, exception):
+    '''
+    record openai chat exception to trace and metrics
+    '''
+    try:
+        duration = time.time() - start_time if "start_time" in locals() else 0
+        if span.is_recording:
+            span.record_exception(exception=exception)
+        record_exception_metric(exception=exception, duration=duration)
+    except Exception as e:
+        logger.warning(f"openai instrument record exception error.{e}")
+
+
+def record_completion(span,
+                      start_time,
+                      response,
+                      request_kwargs,
+                      instance):
+    '''
+    Record chat completion to trace and metrics
+    '''
+    duration = time.time() - start_time if "start_time" in locals() else 0
+    response_dict = model_as_dict(response)
+    attributes = parse_openai_response(
+        response_dict, request_kwargs, instance, False)
+    usage = response_dict.get("usage")
+    choices = response_dict.get("choices")
+    prompt_tokens = usage.get("prompt_tokens")
+    completion_tokens = usage.get("completion_tokens")
+
+    span_attributes = {
+        **attributes,
+        semconv.GEN_AI_USAGE_INPUT_TOKENS: prompt_tokens,
+        semconv.GEN_AI_USAGE_OUTPUT_TOKENS: completion_tokens,
+        semconv.GEN_AI_DURATION: duration
+    }
+    span_attributes.update(parse_response_message(choices))
+    span.set_attributes(span_attributes)
+    record_chat_response_metric(attributes=attributes,
+                                prompt_tokens=prompt_tokens,
+                                completion_tokens=completion_tokens,
+                                duration=duration,
+                                choices=choices
+                                )
+
+
+class WrappedStreamResponse(wrapt.ObjectProxy):
+
+    def __init__(
+        self,
+        span,
+        response,
+        instance=None,
+        start_time=None,
+        request_kwargs=None
+    ):
+        super().__init__(response)
+        self._span = span
+        self._instance = instance
+        self._start_time = start_time
+        self._complete_response = {"choices": [], "model": ""}
+        self._first_token_recorded = False
+        self._request_kwargs = request_kwargs
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.__wrapped__.__exit__(exc_type, exc_val, exc_tb)
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.__wrapped__.__aexit__(exc_type, exc_val, exc_tb)
+
+    def __iter__(self):
+        return self
+
+    def __aiter__(self):
+        return self
+
+    def __next__(self):
+        try:
+            chunk = self.__wrapped__.__next__()
+        except Exception as e:
+            if isinstance(e, StopIteration):
+                self._close_span()
+            raise e
+        else:
+            self._process_stream_chunk(chunk)
+            return chunk
+
+    async def __anext__(self):
+        try:
+            chunk = await self.__wrapped__.__anext__()
+        except Exception as e:
+            if isinstance(e, StopAsyncIteration):
+                self._close_span()
+            raise e
+        else:
+            self._process_stream_chunk(chunk)
+            return chunk
+
+    def _process_stream_chunk(self, chunk):
+        record_stream_response_chunk(chunk, self._complete_response)
+        if not self._first_token_recorded:
+            self._time_of_first_token = time.time()
+            duration = self._time_of_first_token - self._start_time
+            attribute = parse_openai_response(
+                self._complete_response, self._request_kwargs, self._instance, True)
+            record_streaming_time_to_first_token(duration, attribute)
+            self._first_token_recorded = True
+
+    def _close_span(self):
+        duration = None
+        first_token_duration = None
+        first_token_to_generate_duration = None
+        if self._start_time and isinstance(self._start_time, (float, int)):
+            duration = time.time() - self._start_time
+        if self._time_of_first_token and self._start_time and isinstance(self._start_time, (float, int)):
+            first_token_duration = self._time_of_first_token - self._start_time
+            first_token_to_generate_duration = time.time() - self._time_of_first_token
+        prompt_usage, completion_usage = record_stream_token_usage(
+            self._complete_response, self._request_kwargs)
+        attributes = parse_openai_response(
+            self._complete_response, self._request_kwargs, self._instance, True)
+        choices = self._complete_response.get("choices")
+        span_attributes = {
+            **attributes,
+            "llm.prompt_tokens": prompt_usage,
+            "llm.completion_tokens": completion_usage,
+            "llm.duration": duration,
+            "llm.first_token_duration": first_token_duration
+        }
+        span_attributes.update(parse_response_message(choices))
+        self._span.set_attributes(span_attributes)
+        record_chat_response_metric(attributes=attributes,
+                                    prompt_tokens=prompt_usage,
+                                    completion_tokens=completion_usage,
+                                    duration=duration,
+                                    choices=choices
+                                    )
+        record_streaming_time_to_generate(
+            first_token_to_generate_duration, attributes)
+
+        self._span.end()
+
+
+class OpenAIInstrumentor(Instrumentor):
+
+    def instrumentation_dependencies(self) -> Collection[str]:
+        return ("openai >= 1.0.0",)
+
+    def _instrument(self, **kwargs):
+        tracer_provider = kwargs.get("tracer_provider")
+        tracer = tracer_provider.get_tracer(
+            "aworld.trace.instrumentation.openai")
+
+        wrapt.wrap_function_wrapper(
+            "openai.resources.chat.completions",
+            "Completions.create",
+            _chat_wrapper(tracer=tracer)
+        )
+
+        wrapt.wrap_function_wrapper(
+            "openai.resources.chat.completions",
+            "AsyncCompletions.create",
+            _achat_class_wrapper(tracer)
+        )
+
+    def _instrument(self, **kwargs: Any):
+        pass
+
+
+def wrap_openai(client: Union[openai.OpenAI, openai.AsyncOpenAI]):
+    """Patch the OpenAI client to make it traceable.
+       Example:
+       client = wrap_openai(openai.OpenAI())
+    """
+    try:
+        tracer_provider = get_tracer_provider_silent()
+        if not tracer_provider:
+            return
+        tracer = tracer_provider.get_tracer(
+            "aworld.trace.instrumentation.openai")
+
+        if isinstance(client, openai.OpenAI):
+            wrapper = _chat_wrapper(tracer)
+            client.chat.completions.create = wrapper(
+                client.chat.completions.create)
+            logger.info(
+                f"[{client.__class__}]client.chat.completions.create be warpped")
+        if isinstance(client, openai.AsyncOpenAI):
+            awrapper = _achat_instance_wrapper(tracer)
+            client.chat.completions.create = awrapper(
+                client.chat.completions.create)
+            logger.info(
+                f"[{client.__class__}]client.chat.completions.create be warpped")
+    except Exception:
+        logger.warning(traceback.format_exc())
+
+    return client

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/openai/inout_parse.py

@@ -0,0 +1,296 @@
+import asyncio
+import os
+import threading
+import copy
+import json
+import openai
+from importlib.metadata import version
+from aworld.logs.util import logger
+from aworld.trace.base import Span
+from aworld.utils import import_package
+import aworld.trace.instrumentation.semconv as semconv
+
+_PYDANTIC_VERSION = version("pydantic")
+
+
+def should_trace_prompts():
+    '''Determine whether it is necessary to record the message
+    '''
+    return (os.getenv("SHOULD_TRACE_PROMPTS") or "true").lower() == "true"
+
+
+def need_flatten_messages():
+    '''Determine whether it is necessary to flatten the messages
+    '''
+    return (os.getenv("TRACE_FLATTEN_MESSAGES") or "false").lower() == "true"
+
+
+def run_async(method):
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        loop = None
+
+    if loop and loop.is_running():
+        thread = threading.Thread(target=lambda: asyncio.run(method))
+        thread.start()
+        thread.join()
+    else:
+        asyncio.run(method)
+
+
+async def handle_openai_request(span: Span, kwargs, instance):
+    if not span or not span.is_recording():
+        return
+    try:
+        attributes = parser_request_params(kwargs, instance)
+        if should_trace_prompts():
+            messages = kwargs.get("messages")
+            if need_flatten_messages():
+                attributes.update(parse_request_message(messages))
+            else:
+                attributes.update({
+                    semconv.GEN_AI_PROMPT: str(messages),
+                })
+        span.set_attributes(attributes)
+    except ValueError as e:
+        logger.warning(f"trace handle openai request error: {e}")
+
+
+def parser_request_params(kwargs, instance):
+    attributes = {
+        semconv.GEN_AI_SYSTEM: "OpenAI",
+        semconv.GEN_AI_REQUEST_MODEL: kwargs.get("model", ""),
+        semconv.GEN_AI_REQUEST_MAX_TOKENS: kwargs.get("max_tokens", ""),
+        semconv.GEN_AI_REQUEST_TEMPERATURE: kwargs.get("temperature", ""),
+        semconv.GEN_AI_REQUEST_TOP_P: kwargs.get("top_p", ""),
+        semconv.GEN_AI_REQUEST_FREQUENCY_PENALTY: kwargs.get("frequency_penalty", ""),
+        semconv.GEN_AI_REQUEST_PRESENCE_PENALTY: kwargs.get("presence_penalty", ""),
+        semconv.GEN_AI_REQUEST_USER: kwargs.get("user", ""),
+        semconv.GEN_AI_REQUEST_EXTRA_HEADERS: kwargs.get("extra_headers", ""),
+        semconv.GEN_AI_REQUEST_STREAMING: kwargs.get("stream", ""),
+        semconv.GEN_AI_OPERATION_NAME: "chat"
+    }
+
+    client = instance._client
+    if isinstance(client, (openai.AsyncOpenAI, openai.OpenAI)):
+        attributes.update({"llm.base_url": str(client.base_url)})
+
+    filterd_attri = {k: v for k, v in attributes.items()
+                     if (v and v != "")}
+    return filterd_attri
+
+
+def is_streaming_response(response):
+    return isinstance(response, openai.Stream) or isinstance(response, openai.AsyncStream)
+
+
+def parse_openai_response(response, request_kwargs, instance, is_streaming):
+    return {
+        semconv.GEN_AI_RESPONSE_MODEL: response.get("model") or request_kwargs.get("model") or None,
+        semconv.GEN_AI_SERVER_ADDRESS: _get_openai_base_url(instance)
+    }
+
+
+def record_stream_token_usage(complete_response, request_kwargs) -> tuple[int, int]:
+    '''
+        return (prompt_usage, completion_usage)
+    '''
+    prompt_usage = 0
+    completion_usage = 0
+
+    # prompt_usage
+    if request_kwargs and request_kwargs.get("messages"):
+        prompt_content = ""
+        model_name = complete_response.get(
+            "model") or request_kwargs.get("model") or "gpt-4"
+        for msg in request_kwargs.get("messages"):
+            if msg.get("content"):
+                prompt_content += msg.get("content")
+        if model_name:
+            prompt_usage = get_token_count_from_string(
+                prompt_content, model_name)
+
+    # completion_usage
+    if complete_response.get("choices"):
+        completion_content = ""
+        model_name = complete_response.get("model") or "gpt-4"
+
+        for choice in complete_response.get("choices"):
+            if choice.get("message") and choice.get("message").get("content"):
+                completion_content += choice["message"]["content"]
+
+        if model_name:
+            completion_usage = get_token_count_from_string(
+                completion_content, model_name)
+
+    return (prompt_usage, completion_usage)
+
+
+def _get_openai_base_url(instance):
+    if hasattr(instance, "_client"):
+        client = instance._client  # pylint: disable=protected-access
+        if isinstance(client, (openai.AsyncOpenAI, openai.OpenAI)):
+            return str(client.base_url)
+
+    return ""
+
+
+def get_token_count_from_string(string: str, model_name: str):
+    import_package("tiktoken")
+    import tiktoken
+
+    if tiktoken_encodings.get(model_name) is None:
+        try:
+            encoding = tiktoken.encoding_for_model(model_name)
+        except KeyError as ex:
+            logger.warning(
+                f"Failed to get tiktoken encoding for model_name {model_name}, error: {str(ex)}")
+            return None
+
+        tiktoken_encodings[model_name] = encoding
+    else:
+        encoding = tiktoken_encodings.get(model_name)
+
+    token_count = len(encoding.encode(string))
+    return token_count
+
+
+def record_stream_response_chunk(chunk, complete_response):
+    chunk = model_as_dict(chunk)
+    complete_response["model"] = chunk.get("model")
+    complete_response["id"] = chunk.get("id")
+
+    # prompt filter results
+    if chunk.get("prompt_filter_results"):
+        complete_response["prompt_filter_results"] = chunk.get(
+            "prompt_filter_results")
+
+    for choice in chunk.get("choices"):
+        index = choice.get("index")
+        if len(complete_response.get("choices")) <= index:
+            complete_response["choices"].append(
+                {"index": index, "message": {"content": "", "role": ""}})
+        complete_choice = complete_response.get("choices")[index]
+        if choice.get("finish_reason"):
+            complete_choice["finish_reason"] = choice.get("finish_reason")
+        if choice.get("content_filter_results"):
+            complete_choice["content_filter_results"] = choice.get(
+                "content_filter_results")
+
+        delta = choice.get("delta")
+
+        if delta and delta.get("content"):
+            complete_choice["message"]["content"] += delta.get("content")
+
+        if delta and delta.get("role"):
+            complete_choice["message"]["role"] = delta.get("role")
+        if delta and delta.get("tool_calls"):
+            tool_calls = delta.get("tool_calls")
+            if not isinstance(tool_calls, list) or len(tool_calls) == 0:
+                continue
+
+            if not complete_choice["message"].get("tool_calls"):
+                complete_choice["message"]["tool_calls"] = []
+
+            for tool_call in tool_calls:
+                i = int(tool_call["index"])
+                if len(complete_choice["message"]["tool_calls"]) <= i:
+                    complete_choice["message"]["tool_calls"].append(
+                        {"id": "", "function": {"name": "", "arguments": ""}}
+                    )
+
+                span_tool_call = complete_choice["message"]["tool_calls"][i]
+                span_function = span_tool_call["function"]
+                tool_call_function = tool_call.get("function")
+
+                if tool_call.get("id"):
+                    span_tool_call["id"] = tool_call.get("id")
+                if tool_call_function and tool_call_function.get("name"):
+                    span_function["name"] = tool_call_function.get("name")
+                if tool_call_function and tool_call_function.get("arguments"):
+                    span_function["arguments"] += tool_call_function.get(
+                        "arguments")
+
+
+def parse_request_message(messages):
+    '''
+    flatten request message to attributes
+    '''
+    attributes = {}
+    for i, msg in enumerate(messages):
+        prefix = f"{semconv.GEN_AI_PROMPT}.{i}"
+        attributes.update({f"{prefix}.role": msg.get("role")})
+        if msg.get("content"):
+            content = copy.deepcopy(msg.get("content"))
+            content = json.dumps(content)
+            attributes.update({f"{prefix}.content": content})
+        if msg.get("tool_call_id"):
+            attributes.update({
+                f"{prefix}.tool_call_id": msg.get("tool_call_id")})
+        tool_calls = msg.get("tool_calls")
+        if tool_calls:
+            for i, tool_call in enumerate(tool_calls):
+                tool_call = model_as_dict(tool_call)
+                function = tool_call.get("function")
+                attributes.update({
+                    f"{prefix}.tool_calls.{i}.id": tool_call.get("id")})
+                attributes.update({
+                    f"{prefix}.tool_calls.{i}.name": function.get("name")})
+                attributes.update({
+                    f"{prefix}.tool_calls.{i}.arguments": function.get("arguments")})
+    return attributes
+
+
+def parse_response_message(choices) -> dict:
+    attributes = {}
+    if not should_trace_prompts():
+        return attributes
+    for choice in choices:
+        index = choice.get("index")
+        prefix = f"{semconv.GEN_AI_COMPLETION}.{index}"
+        attributes.update(
+            {f"{prefix}.finish_reason": choice.get("finish_reason")})
+
+        message = choice.get("message")
+        if not message:
+            continue
+
+        attributes.update({f"{prefix}.role": message.get("role")})
+
+        if message.get("refusal"):
+            attributes.update({f"{prefix}.refusal": message.get("refusal")})
+        else:
+            attributes.update({f"{prefix}.content": message.get("content")})
+
+        function_call = message.get("function_call")
+        if function_call:
+            attributes.update(
+                {f"{prefix}.tool_calls.0.name": function_call.get("name")})
+            attributes.update(
+                {f"{prefix}.tool_calls.0.arguments": function_call.get("arguments")})
+
+        tool_calls = message.get("tool_calls")
+        if tool_calls:
+            for i, tool_call in enumerate(tool_calls):
+                function = tool_call.get("function")
+                attributes.update(
+                    {f"{prefix}.tool_calls.{i}.id": tool_call.get("id")})
+                attributes.update(
+                    {f"{prefix}.tool_calls.{i}.name": function.get("name")})
+                attributes.update(
+                    {f"{prefix}.tool_calls.{i}.arguments": function.get("arguments")})
+    return attributes
+
+
+def model_as_dict(model):
+    if isinstance(model, dict):
+        return model
+    if _PYDANTIC_VERSION < "2.0.0":
+        return model.dict()
+    if hasattr(model, "model_dump"):
+        return model.model_dump()
+    elif hasattr(model, "parse"):  # Raw API response
+        return model_as_dict(model.parse())
+    else:
+        return model

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/requests.py

@@ -0,0 +1,213 @@
+from aworld.logs.util import logger
+from aworld.trace.instrumentation.http_util import (
+    collect_attributes_from_request,
+    url_disabled,
+    get_excluded_urls,
+    parse_excluded_urls,
+    HTTP_RESPONSE_STATUS_CODE,
+    HTTP_FLAVOR
+)
+from aworld.metrics.context_manager import MetricContext
+from aworld.metrics.template import MetricTemplate
+from aworld.metrics.metric import MetricType
+from aworld.trace.instrumentation import Instrumentor
+from aworld.trace.propagator.carrier import DictCarrier
+import functools
+from timeit import default_timer
+from requests import sessions
+from requests.models import PreparedRequest, Response
+from requests.structures import CaseInsensitiveDict
+from typing import Collection, Any, Callable
+from aworld.trace.base import TraceProvider, TraceContext, Tracer, SpanType, get_tracer_provider
+from aworld.trace.propagator import get_global_trace_propagator
+
+
+def _wrapped_send(
+    tracer: Tracer = None,
+    excluded_urls=None,
+    request_hook: Callable = None,
+    response_hook: Callable = None,
+    duration_histogram: MetricTemplate = None
+):
+
+    oringinal_send = sessions.Session.send
+
+    @functools.wraps(oringinal_send)
+    def instrumented_send(
+        self: sessions.Session, request: PreparedRequest, **kwargs: Any
+    ):
+        if excluded_urls and url_disabled(request.url, excluded_urls):
+            return oringinal_send(self, request, **kwargs)
+
+        def get_or_create_headers():
+            request.headers = (
+                request.headers
+                if request.headers is not None
+                else CaseInsensitiveDict()
+            )
+            return request.headers
+
+        method = request.method
+        if method is None:
+            method = "HTTP"
+        span_name = method.upper()
+
+        span_attributes = collect_attributes_from_request(request)
+        with tracer.start_as_current_span(
+            span_name, span_type=SpanType.CLIENT, attributes=span_attributes
+        ) as span:
+            exception = None
+            if callable(request_hook):
+                request_hook(span, request)
+
+            headers = get_or_create_headers()
+
+            trace_context = TraceContext(
+                trace_id=span.get_trace_id(),
+                span_id=span.get_span_id(),
+            )
+            propagator = get_global_trace_propagator()
+            if propagator:
+                propagator.inject(trace_context, DictCarrier(headers))
+
+            start_time = default_timer()
+            try:
+                logger.info("Sending headers: %s", request.headers)
+                result = oringinal_send(
+                    self, request, **kwargs
+                )  # *** PROCEED
+            except Exception as exc:  # pylint: disable=W0703
+                exception = exc
+                result = getattr(exc, "response", None)
+            finally:
+                elapsed_time = max(default_timer() - start_time, 0)
+
+            if isinstance(result, Response):
+                span_attributes = {}
+                span_attributes[HTTP_RESPONSE_STATUS_CODE] = result.status_code
+
+                if result.raw is not None:
+                    version = getattr(result.raw, "version", None)
+                    if version:
+                        # Only HTTP/1 is supported by requests
+                        version_text = "1.1" if version == 11 else "1.0"
+                        span_attributes[HTTP_FLAVOR] = version_text
+                span.set_attributes(span_attributes)
+
+                if callable(response_hook):
+                    response_hook(span, request, result)
+
+            if exception is not None:
+                span.record_exception(exception)
+
+            if duration_histogram is not None and MetricContext.metric_initialized():
+                MetricContext.histogram_record(
+                    duration_histogram,
+                    elapsed_time,
+                    span_attributes
+                )
+
+            if exception is not None:
+                raise exception.with_traceback(exception.__traceback__)
+
+        return result
+
+    return instrumented_send
+
+
+class _InstrumentedSession(sessions.Session):
+    """
+    An instrumented requests.Session class.
+    """
+    _excluded_urls = None
+    _tracer_provider: TraceProvider = None
+    _request_hook = None
+    _response_hook = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        tracer = self._tracer_provider.get_tracer(
+            "aworld.trace.instrumentation.requests")
+        excluded_urls = kwargs.get("excluded_urls")
+
+        duration_histogram = MetricTemplate(
+            type=MetricType.HISTOGRAM,
+            name="client_request_duration_histogram",
+            unit="s",
+            description="Duration of  HTTP client requests."
+        )
+        self.send = functools.partial(_wrapped_send(
+            tracer=tracer,
+            excluded_urls=excluded_urls,
+            request_hook=self._request_hook,
+            response_hook=self._response_hook,
+            duration_histogram=duration_histogram
+        ), self)
+
+
+class RequestsInstrumentor(Instrumentor):
+    """
+    An instrumentor for the requests module.
+    """
+
+    def instrumentation_dependencies(self) -> Collection[str]:
+        return ["requests"]
+
+    def _instrument(self, **kwargs):
+        """
+        Instruments the requests module.
+        """
+        logger.info("requests _instrument entered.")
+        self._original_session = sessions.Session
+        request_hook = kwargs.get("request_hook")
+        response_hook = kwargs.get("response_hook")
+        if callable(request_hook):
+            _InstrumentedSession._request_hook = request_hook
+        if callable(response_hook):
+            _InstrumentedSession._response_hook = response_hook
+        tracer_provider = kwargs.get("tracer_provider")
+        _InstrumentedSession._tracer_provider = tracer_provider
+        excluded_urls = kwargs.get("excluded_urls")
+        _InstrumentedSession._excluded_urls = (
+            get_excluded_urls("FLASK")
+            if excluded_urls is None
+            else parse_excluded_urls(excluded_urls)
+        )
+        sessions.Session = _InstrumentedSession
+        logger.info("requests _instrument exited.")
+
+    def _uninstrument(self, **kwargs):
+        """
+        Uninstruments the requests module.
+        """
+        sessions.Session = self._original_session
+
+
+def instrument_requests(excluded_urls: str = None,
+                        request_hook: Callable = None,
+                        response_hook: Callable = None,
+                        tracer_provider: TraceProvider = None,
+                        **kwargs: Any,
+                        ):
+    """
+    Instruments the requests module.
+    Args:
+        excluded_urls: A comma separated list of URLs to exclude from tracing.
+        request_hook: A function that will be called before a request is sent.  
+            The function will be called with the span and the request.
+        response_hook: A function that will be called after a response is received.
+            The function will be called with the span and the response.
+        tracer_provider: The tracer provider to use. If not provided, the global
+            tracer provider will be used.
+        kwargs: Additional keyword arguments.
+    """
+    all_kwargs = {
+        "excluded_urls": excluded_urls,
+        "request_hook": request_hook,
+        "response_hook": response_hook,
+        "tracer_provider": tracer_provider or get_tracer_provider(),
+        **kwargs
+    }
+    RequestsInstrumentor().instrument(**all_kwargs)
+    logger.info("Requests instrumented.")

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/semconv.py

@@ -0,0 +1,46 @@
+# GenAI semconv attribute names
+import os
+
+GEN_AI_SYSTEM = "gen_ai.system"
+GEN_AI_REQUEST_MODEL = "gen_ai.request.model"
+GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty"
+GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"
+GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty"
+GEN_AI_REQUEST_STOP_SEQUENCES = "gen_ai.request.stop_sequences"
+GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"
+GEN_AI_REQUEST_TOP_K = "gen_ai.request.top_k"
+GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p"
+GEN_AI_REQUEST_STREAMING = "gen_ai.request.streaming"
+GEN_AI_REQUEST_USER = "gen_ai.request.user"
+GEN_AI_REQUEST_EXTRA_HEADERS = "gen_ai.request.extra_headers"
+GEN_AI_PROMPT = "gen_ai.prompt"
+GEN_AI_PROMPT_TOOLS = "gen_ai.prompt.tools"
+GEN_AI_COMPLETION = "gen_ai.completion"
+GEN_AI_COMPLETION_TOOL_CALLS = "gen_ai.completion.tool_calls"
+GEN_AI_COMPLETION_CONTENT = "gen_ai.completion.content"
+GEN_AI_DURATION = "gen_ai.duration"
+GEN_AI_FIRST_TOKEN_DURATION = "gen_ai.first_token_duration"
+GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons"
+GEN_AI_RESPONSE_ID = "gen_ai.response.id"
+GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"
+GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"
+GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"
+GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens"
+GEN_AI_OPERATION_NAME = "gen_ai.operation.name"
+GEN_AI_METHOD_NAME = "gen_ai.method.name"
+GEN_AI_SERVER_ADDRESS = "gen_ai.server.address"
+
+ATTRIBUTE_NAME_SPACE = os.getenv("ATTRIBUTE_NAME_SPACE", "aworld.")
+AGENT_ID = ATTRIBUTE_NAME_SPACE + "agent.id"
+AGENT_NAME = ATTRIBUTE_NAME_SPACE + "agent.name"
+AGENT_RUN_SUCCESS = ATTRIBUTE_NAME_SPACE + "agent.run.success"
+AGENT_USAGE_TYPE = ATTRIBUTE_NAME_SPACE + "agent.usage_type"
+TOOL_NAME = ATTRIBUTE_NAME_SPACE + "tool.name"
+TOOL_STEP_SUCCESS = ATTRIBUTE_NAME_SPACE + "tool.step.success"
+TASK = ATTRIBUTE_NAME_SPACE + "task"
+TASK_ID = ATTRIBUTE_NAME_SPACE + "task.id"
+TASK_INPUT = ATTRIBUTE_NAME_SPACE + "task.input"
+TASK_IS_SUB_TASK = ATTRIBUTE_NAME_SPACE + "task.is_sub_task"
+TASK_GROUP_ID = ATTRIBUTE_NAME_SPACE + "task.group_id"
+SESSION_ID = ATTRIBUTE_NAME_SPACE + "session.id"
+USER_ID = ATTRIBUTE_NAME_SPACE + "user.id"

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/threading.py

@@ -0,0 +1,149 @@
+import threading
+from typing import Protocol, TypeVar, Any, Callable
+from wrapt import wrap_function_wrapper
+from concurrent import futures
+import aworld.trace as trace
+from aworld.trace.base import TraceContext, Span
+from aworld.trace.propagator import get_global_trace_context
+from aworld.trace.instrumentation import Instrumentor
+from aworld.trace.instrumentation.utils import unwrap
+from aworld.logs.util import logger
+
+
+R = TypeVar("R")
+
+
+class HasTraceContext(Protocol):
+    _trace_context: TraceContext
+
+
+class ThreadingInstrumentor(Instrumentor):
+    '''
+    Trace instrumentor for threading
+    '''
+
+    def instrumentation_dependencies(self) -> str:
+        return ()
+
+    def _instrument(self, **kwargs: Any):
+        self._instrument_thread()
+        self._instrument_timer()
+        self._instrument_thread_pool()
+
+    def _uninstrument(self, **kwargs: Any):
+        self._uninstrument_thread()
+        self._uninstrument_timer()
+        self._uninstrument_thread_pool()
+
+    @staticmethod
+    def _instrument_thread():
+        wrap_function_wrapper(
+            threading.Thread,
+            "start",
+            ThreadingInstrumentor.__wrap_threading_start,
+        )
+        wrap_function_wrapper(
+            threading.Thread,
+            "run",
+            ThreadingInstrumentor.__wrap_threading_run,
+        )
+
+    @staticmethod
+    def _instrument_timer():
+        wrap_function_wrapper(
+            threading.Timer,
+            "start",
+            ThreadingInstrumentor.__wrap_threading_start,
+        )
+        wrap_function_wrapper(
+            threading.Timer,
+            "run",
+            ThreadingInstrumentor.__wrap_threading_run,
+        )
+
+    @staticmethod
+    def _instrument_thread_pool():
+        wrap_function_wrapper(
+            futures.ThreadPoolExecutor,
+            "submit",
+            ThreadingInstrumentor.__wrap_thread_pool_submit,
+        )
+
+    @staticmethod
+    def _uninstrument_thread():
+        unwrap(threading.Thread, "start")
+        unwrap(threading.Thread, "run")
+
+    @staticmethod
+    def _uninstrument_timer():
+        unwrap(threading.Timer, "start")
+        unwrap(threading.Timer, "run")
+
+    @staticmethod
+    def _uninstrument_thread_pool():
+        unwrap(futures.ThreadPoolExecutor, "submit")
+
+    @staticmethod
+    def __wrap_threading_start(
+        call_wrapped: Callable[[], None],
+        instance: HasTraceContext,
+        args: tuple[()],
+        kwargs: dict[str, Any],
+    ) -> None:
+        span: Span = trace.get_current_span()
+        if span:
+            instance._trace_context = TraceContext(
+                trace_id=span.get_trace_id(), span_id=span.get_span_id())
+        return call_wrapped(*args, **kwargs)
+
+    @staticmethod
+    def __wrap_threading_run(
+        call_wrapped: Callable[..., R],
+        instance: HasTraceContext,
+        args: tuple[Any, ...],
+        kwargs: dict[str, Any],
+    ) -> R:
+
+        token = None
+        try:
+            if hasattr(instance, "_trace_context"):
+                if instance._trace_context:
+                    token = get_global_trace_context().set(instance._trace_context)
+            return call_wrapped(*args, **kwargs)
+        finally:
+            if token:
+                get_global_trace_context().reset(token)
+
+    @staticmethod
+    def __wrap_thread_pool_submit(
+        call_wrapped: Callable[..., R],
+        instance: futures.ThreadPoolExecutor,
+        args: tuple[Callable[..., Any], ...],
+        kwargs: dict[str, Any],
+    ) -> R:
+        # obtain the original function and wrapped kwargs
+        original_func = args[0]
+        trace_context = None
+        span: Span = trace.get_current_span()
+        if span and span.get_trace_id() != "":
+            trace_context = TraceContext(
+                trace_id=span.get_trace_id(), span_id=span.get_span_id())
+
+        def wrapped_func(*func_args: Any, **func_kwargs: Any) -> R:
+            token = None
+            try:
+                if trace_context:
+                    token = get_global_trace_context().set(trace_context)
+                return original_func(*func_args, **func_kwargs)
+            finally:
+                if token:
+                    get_global_trace_context().reset(token)
+
+        # replace the original function with the wrapped function
+        new_args: tuple[Callable[..., Any], ...] = (wrapped_func,) + args[1:]
+        return call_wrapped(*new_args, **kwargs)
+
+
+def instrument_theading(**kwargs: Any) -> None:
+    ThreadingInstrumentor().instrument(**kwargs)
+    logger.info("Threading instrumented")

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/tool/__init__.py

@@ -0,0 +1,249 @@
+from asyncio import iscoroutine
+import wrapt
+import time
+import traceback
+import aworld.trace.constants as trace_constants
+from typing import Collection, Any, Union, Sequence
+from aworld.trace.instrumentation import Instrumentor
+from aworld.trace.instrumentation import semconv
+from aworld.trace.base import (
+    Tracer,
+    SpanType,
+    get_tracer_provider_silent
+)
+from aworld.logs.util import logger
+from aworld.metrics.context_manager import MetricContext
+from aworld.metrics.template import MetricTemplate
+from aworld.metrics.metric import MetricType
+
+tool_duration_histogram = MetricTemplate(
+    type=MetricType.HISTOGRAM,
+    name="tool_step_duration",
+    unit="s",
+    description="tool step run duration",
+)
+
+tool_step_counter = MetricTemplate(
+    type=MetricType.COUNTER,
+    name="tool_step_counter",
+    unit="time",
+    description="Number of tool step run",
+)
+
+
+def get_tool_name(tool_name: str,
+                  action: Union['ActionModel', Sequence['ActionModel']]) -> tuple[str, trace_constants.RunType]:
+    if tool_name == "mcp" and action:
+        try:
+            if isinstance(action, (list, tuple)):
+                action = action[0]
+            mcp_name = action.action_name.split("__")[0]
+            return (mcp_name, trace_constants.RunType.MCP)
+        except ValueError:
+            logger.warning(traceback.format_exc())
+            return (tool_name, trace_constants.RunType.MCP)
+    return (tool_name, trace_constants.RunType.TOOL)
+
+
+def get_tool_span_attributes(instance, message: 'Message'):
+    run_type = trace_constants.RunType.TOOL
+    action = message.payload
+    agent_id = None
+    tool_name = instance.name()
+    if isinstance(action, (list, tuple)):
+        action = action[0]
+        if action:
+            agent_id = action.agent_name
+            tool_name, run_type = get_tool_name(action.tool_name, action)
+    return {
+        semconv.TOOL_NAME: tool_name,
+        semconv.AGENT_ID: agent_id,
+        semconv.AGENT_NAME: _get_agent_name_from_id(agent_id),
+        semconv.TASK_ID: message.context.task_id if (message.context and message.context.task_id) else "",
+        semconv.SESSION_ID: message.context.session_id if (message.context and message.context.session_id) else "",
+        semconv.USER_ID: message.context.user if (message.context and message.context.user) else "",
+        trace_constants.ATTRIBUTES_MESSAGE_RUN_TYPE_KEY: run_type.value
+    }
+
+
+def _end_span(span):
+    if span:
+        span.end()
+
+
+def _get_agent_name_from_id(agent_id):
+    if agent_id and '---' in agent_id:
+        return agent_id.split('---', 1)[0]
+    return agent_id
+
+
+def _record_metric(duration, attributes, exception=None):
+    if MetricContext.metric_initialized():
+        MetricContext.histogram_record(tool_duration_histogram, duration, labels=attributes)
+        if exception:
+            run_counter_attr = {
+                semconv.TOOL_STEP_SUCCESS: "0",
+                "error.type": exception.__class__.__name__,
+                **attributes
+            }
+        else:
+            run_counter_attr = {
+                semconv.TOOL_STEP_SUCCESS: "1",
+                **attributes
+            }
+        MetricContext.count(tool_step_counter, 1, labels=run_counter_attr)
+
+
+def _record_exception(span, start_time, exception, attributes):
+    try:
+        duration = time.time() - start_time if "start_time" in locals() else 0
+        if span.is_recording:
+            span.record_exception(exception=exception)
+        _record_metric(duration, attributes, exception)
+    except Exception as e:
+        logger.warning(f"tool instrument record exception error.{e}")
+
+
+def _record_response(instance,
+                     start_time,
+                     response,
+                     attributes):
+    try:
+        duration = time.time() - start_time if "start_time" in locals() else 0
+        _record_metric(duration, attributes)
+    except Exception as e:
+        logger.warning(f"tool instrument record response error.{e}")
+
+
+def _async_step_class_wrapper(tracer: Tracer):
+    async def _async_step_wrapper(wrapped, instance, args, kwargs):
+        span = None
+        message = args[0] or kwargs.get("message")
+        attributes = get_tool_span_attributes(instance, message)
+        if tracer:
+            span = tracer.start_span(
+                name=trace_constants.SPAN_NAME_PREFIX_TOOL + "step",
+                span_type=SpanType.SERVER,
+                attributes=attributes
+            )
+        start_time = time.time()
+        try:
+            response = await wrapped(*args, **kwargs)
+            _record_response(instance, start_time, response, attributes)
+        except Exception as e:
+            _record_exception(span=span,
+                              start_time=start_time,
+                              exception=e,
+                              attributes=attributes
+                              )
+            _end_span(span)
+            raise e
+        _end_span(span)
+        return response
+    return _async_step_wrapper
+
+
+def _step_class_wrapper(tracer: Tracer):
+    def _step_wrapper(wrapped, instance, args, kwargs):
+        span = None
+        message = args[0] or kwargs.get("message")
+        attributes = get_tool_span_attributes(instance, message)
+        if tracer:
+            span = tracer.start_span(
+                name=trace_constants.SPAN_NAME_PREFIX_TOOL + "step",
+                span_type=SpanType.SERVER,
+                attributes=attributes
+            )
+        start_time = time.time()
+        try:
+            response = wrapped(*args, **kwargs)
+            _record_response(instance, start_time, response, attributes)
+        except Exception as e:
+            _record_exception(span=span,
+                              start_time=start_time,
+                              exception=e,
+                              attributes=attributes
+                              )
+            _end_span(span)
+            raise e
+        _end_span(span)
+        return response
+    return _step_wrapper
+
+async def _async_step_instance_wrapper(tracer: Tracer):
+
+    @wrapt.decorator
+    async def _awrapper(wrapped, instance, args, kwargs):
+        wrapper_func = _async_step_class_wrapper(tracer=tracer)
+        return await wrapper_func(wrapped, instance, args, kwargs)
+
+    return _awrapper
+
+def _step_instance_wrapper(tracer: Tracer):
+
+    @wrapt.decorator
+    def _wrapper(wrapped, instance, args, kwargs):
+        wrapper_func = _step_class_wrapper(tracer)
+        return wrapper_func(wrapped, instance, args, kwargs)
+
+    return _wrapper
+
+class ToolInstrumentor(Instrumentor):
+
+    def instrumentation_dependencies(self) -> Collection[str]:
+        return ()
+
+    def _instrument(self, **kwargs):
+        agent_trace_enabled = kwargs.get("trace_enabled", False)
+        tracer_provider = get_tracer_provider_silent()
+        tracer = None
+        if tracer_provider and agent_trace_enabled:
+            tracer = tracer_provider.get_tracer(
+                "aworld.trace.instrumentation.tool")
+
+        wrapt.wrap_function_wrapper(
+            "aworld.core.tool.base",
+            "AsyncBaseTool.step",
+            _async_step_class_wrapper(tracer=tracer)
+        )
+
+        wrapt.wrap_function_wrapper(
+            "aworld.core.tool.base",
+            "AsyncTool.step",
+            _async_step_class_wrapper(tracer=tracer)
+        )
+
+        wrapt.wrap_function_wrapper(
+            "aworld.core.tool.base",
+            "BaseTool.step",
+            _step_class_wrapper(tracer=tracer)
+        )
+
+        wrapt.wrap_function_wrapper(
+            "aworld.core.tool.base",
+            "Tool.step",
+            _step_class_wrapper(tracer=tracer)
+        )
+
+    def _uninstrument(self, **kwargs: Any):
+        pass
+
+
+def wrap_tool(tool):
+    try:
+        tracer_provider = get_tracer_provider_silent()
+        if not tracer_provider:
+            return tool
+        tracer = tracer_provider.get_tracer(
+            "aworld.trace.instrumentation.tool")
+
+        async_wrapper = _async_step_instance_wrapper(tracer)
+        wrapper = _step_instance_wrapper(tracer)
+        if iscoroutine(tool.step):
+            tool.step = async_wrapper(tool.step)
+        else:
+            tool.step = wrapper(tool.step)
+    except Exception:
+        logger.warning(traceback.format_exc())
+
+    return tool

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/uni_llmmodel/__init__.py

@@ -0,0 +1,379 @@
+import wrapt
+import time
+import traceback
+import aworld.trace.instrumentation.semconv as semconv
+from typing import Collection, Any
+from aworld.trace.instrumentation import Instrumentor
+from aworld.trace.base import (
+    Tracer,
+    SpanType,
+    get_tracer_provider_silent
+)
+from aworld.trace.constants import ATTRIBUTES_MESSAGE_RUN_TYPE_KEY, RunType, SPAN_NAME_PREFIX_LLM
+from aworld.trace.instrumentation.llm_metrics import (
+    record_exception_metric,
+    record_chat_response_metric,
+    record_streaming_time_to_first_token,
+    record_streaming_time_to_generate
+)
+from aworld.trace.instrumentation.uni_llmmodel.model_response_parse import (
+    accumulate_stream_response,
+    get_common_attributes_from_response,
+    record_stream_token_usage,
+    parse_response_message,
+    response_to_dic,
+    handle_request
+)
+from aworld.trace.instrumentation.openai.inout_parse import run_async
+
+from aworld.models.model_response import ModelResponse
+from aworld.logs.util import logger
+
+
+def _completion_class_wrapper(tracer: Tracer):
+
+    def wrapper(wrapped, instance, args, kwargs):
+        model_name = instance.provider.model_name
+        if not model_name:
+            model_name = "LLMModel"
+        span_attributes = {}
+        span_attributes[ATTRIBUTES_MESSAGE_RUN_TYPE_KEY] = RunType.LLM.value
+
+        span = tracer.start_span(
+            name=SPAN_NAME_PREFIX_LLM + model_name, span_type=SpanType.CLIENT, attributes=span_attributes)
+
+        run_async(handle_request(span, kwargs, instance))
+        start_time = time.time()
+        try:
+            response = wrapped(*args, **kwargs)
+        except Exception as e:
+            record_exception(span=span,
+                             start_time=start_time,
+                             exception=e
+                             )
+            span.end()
+            raise e
+
+        record_completion(span=span,
+                          start_time=start_time,
+                          response=response,
+                          request_kwargs=kwargs,
+                          instance=instance,
+                          is_async=False
+                          )
+        span.end()
+        return response
+
+    return wrapper
+
+
+def _completion_instance_wrapper(tracer: Tracer):
+
+    @wrapt.decorator
+    def _wrapper(wrapped, instance, args, kwargs):
+        wrapper_func = _completion_class_wrapper(tracer)
+        return wrapper_func(wrapped, instance, args, kwargs)
+
+    return _wrapper
+
+
+def _stream_completion_class_wrapper(tracer: Tracer):
+    def wrapper(wrapped, instance, args, kwargs):
+        model_name = instance.provider.model_name
+        if not model_name:
+            model_name = "LLMModel"
+        span_attributes = {}
+        span_attributes[ATTRIBUTES_MESSAGE_RUN_TYPE_KEY] = RunType.LLM.value
+
+        span = tracer.start_span(
+            name=SPAN_NAME_PREFIX_LLM + model_name, span_type=SpanType.CLIENT, attributes=span_attributes)
+
+        run_async(handle_request(span, kwargs, instance))
+        start_time = time.time()
+        try:
+            response = wrapped(*args, **kwargs)
+        except Exception as e:
+            record_exception(span=span,
+                             start_time=start_time,
+                             exception=e
+                             )
+            span.end()
+            raise e
+        return WrappedGeneratorResponse(span=span,
+                                        response=response,
+                                        instance=instance,
+                                        start_time=start_time,
+                                        request_kwargs=kwargs
+                                        )
+    return wrapper
+
+
+def _stream_completion_instance_wrapper(tracer: Tracer):
+
+    @wrapt.decorator
+    def _stream_wrapper(wrapped, instance, args, kwargs):
+        wrapper_func = _stream_completion_class_wrapper(tracer)
+        return wrapper_func(wrapped, instance, args, kwargs)
+
+    return _stream_wrapper
+
+
+def _acompletion_class_wrapper(tracer: Tracer):
+
+    async def awrapper(wrapped, instance, args, kwargs):
+        model_name = instance.provider.model_name
+        if not model_name:
+            model_name = "LLMModel"
+        span_attributes = {}
+        span_attributes[ATTRIBUTES_MESSAGE_RUN_TYPE_KEY] = RunType.LLM.value
+
+        span = tracer.start_span(
+            name=SPAN_NAME_PREFIX_LLM + model_name, span_type=SpanType.CLIENT, attributes=span_attributes)
+
+        await handle_request(span, kwargs, instance)
+        start_time = time.time()
+        try:
+            response = await wrapped(*args, **kwargs)
+        except Exception as e:
+            record_exception(span=span,
+                             start_time=start_time,
+                             exception=e
+                             )
+            span.end()
+            raise e
+
+        record_completion(span=span,
+                          start_time=start_time,
+                          response=response,
+                          request_kwargs=kwargs,
+                          instance=instance,
+                          is_async=True
+                          )
+        span.end()
+        return response
+
+    return awrapper
+
+
+async def _acompletion_instance_wrapper(tracer: Tracer):
+
+    @wrapt.decorator
+    async def _awrapper(wrapped, instance, args, kwargs):
+        wrapper_func = _acompletion_class_wrapper(tracer)
+        return await wrapper_func(wrapped, instance, args, kwargs)
+
+    return _awrapper
+
+
+def record_exception(span, start_time, exception):
+    '''
+    record openai chat exception to trace and metrics
+    '''
+    try:
+        duration = time.time() - start_time if "start_time" in locals() else 0
+        if span.is_recording:
+            span.record_exception(exception=exception)
+        record_exception_metric(exception=exception, duration=duration)
+    except Exception as e:
+        logger.warning(f"openai instrument record exception error.{e}")
+
+
+def record_completion(span,
+                      start_time,
+                      response,
+                      request_kwargs,
+                      instance,
+                      is_async):
+    '''
+    Record chat completion to trace and metrics
+    '''
+    duration = time.time() - start_time if "start_time" in locals() else 0
+    response_dict = response_to_dic(response)
+    attributes = get_common_attributes_from_response(instance, is_async, False)
+    usage = response_dict.get("usage")
+    content = response_dict.get("content", "")
+    tool_calls = response_dict.get("tool_calls")
+    prompt_tokens = -1
+    completion_tokens = -1
+    total_tokens = -1
+    if usage:
+        prompt_tokens = usage.get("prompt_tokens")
+        completion_tokens = usage.get("completion_tokens")
+        total_tokens = usage.get("total_tokens")
+
+    span_attributes = {
+        **attributes,
+        semconv.GEN_AI_USAGE_INPUT_TOKENS: prompt_tokens,
+        semconv.GEN_AI_USAGE_OUTPUT_TOKENS: completion_tokens,
+        semconv.GEN_AI_USAGE_TOTAL_TOKENS: total_tokens,
+        semconv.GEN_AI_DURATION: duration,
+        semconv.GEN_AI_COMPLETION_CONTENT: content
+    }
+    span_attributes.update(parse_response_message(tool_calls))
+    span.set_attributes(span_attributes)
+    record_chat_response_metric(attributes=attributes,
+                                prompt_tokens=prompt_tokens,
+                                completion_tokens=completion_tokens,
+                                duration=duration
+                                )
+
+
+class WrappedGeneratorResponse():
+
+    def __init__(
+        self,
+        span,
+        response,
+        instance=None,
+        start_time=None,
+        request_kwargs=None
+    ):
+        self._span = span
+        self._response = response
+        self._instance = instance
+        self._start_time = start_time
+        self._complete_response = {
+            "id": "", "model": "", "content": "", "tool_calls": [], "usage": {}}
+        self._first_token_recorded = False
+        self._time_of_first_token = None
+        self._request_kwargs = request_kwargs
+
+    def __iter__(self):
+        return self
+
+    def __aiter__(self):
+        return self
+
+    def __next__(self):
+        try:
+            chunk = self._response.__next__()
+        except Exception as e:
+            if isinstance(e, StopIteration):
+                self._close_span(False)
+            raise e
+        else:
+            self._process_stream_chunk(chunk, False)
+            return chunk
+
+    async def __anext__(self):
+        try:
+            chunk = await self._response.__anext__()
+        except Exception as e:
+            if isinstance(e, StopAsyncIteration):
+                self._close_span(True)
+            raise e
+        else:
+            self._process_stream_chunk(chunk, True)
+            return chunk
+
+    def _process_stream_chunk(self, chunk: ModelResponse, is_async):
+        accumulate_stream_response(chunk, self._complete_response)
+
+        if not self._first_token_recorded:
+            self._time_of_first_token = time.time()
+            duration = self._time_of_first_token - self._start_time
+            attribute = get_common_attributes_from_response(
+                self._instance, is_async, True)
+            record_streaming_time_to_first_token(duration, attribute)
+            self._first_token_recorded = True
+
+    def _close_span(self, is_async):
+        duration = None
+        first_token_duration = None
+        first_token_to_generate_duration = None
+        if self._start_time and isinstance(self._start_time, (float, int)):
+            duration = time.time() - self._start_time
+        if self._time_of_first_token and self._start_time and isinstance(self._start_time, (float, int)):
+            first_token_duration = self._time_of_first_token - self._start_time
+            first_token_to_generate_duration = time.time() - self._time_of_first_token
+
+        prompt_usage, completion_usage = record_stream_token_usage(
+            self._complete_response, self._request_kwargs)
+
+        attributes = get_common_attributes_from_response(
+            self._instance, is_async, True)
+
+        span_attributes = {
+            **attributes,
+            semconv.GEN_AI_USAGE_INPUT_TOKENS: prompt_usage,
+            semconv.GEN_AI_USAGE_OUTPUT_TOKENS: completion_usage,
+            semconv.GEN_AI_USAGE_TOTAL_TOKENS: prompt_usage + completion_usage,
+            semconv.GEN_AI_DURATION: duration,
+            semconv.GEN_AI_FIRST_TOKEN_DURATION: first_token_duration,
+            semconv.GEN_AI_COMPLETION_CONTENT: self._complete_response.get(
+                "content", "")
+        }
+        span_attributes.update(parse_response_message(
+            self._complete_response.get("tool_calls", [])))
+
+        self._span.set_attributes(span_attributes)
+        record_chat_response_metric(attributes=attributes,
+                                    prompt_tokens=prompt_usage,
+                                    completion_tokens=completion_usage,
+                                    duration=duration
+                                    )
+        record_streaming_time_to_generate(
+            first_token_to_generate_duration, attributes)
+
+        self._span.end()
+
+
+class LLMModelInstrumentor(Instrumentor):
+
+    def instrumentation_dependencies(self) -> Collection[str]:
+        return ()
+
+    def _instrument(self, **kwargs):
+        tracer_provider = get_tracer_provider_silent()
+        if not tracer_provider:
+            return
+        tracer = tracer_provider.get_tracer(
+            "aworld.trace.instrumentation.llmmodel")
+
+        wrapt.wrap_function_wrapper(
+            "aworld.models.llm",
+            "LLMModel.completion",
+            _completion_class_wrapper(tracer=tracer)
+        )
+
+        wrapt.wrap_function_wrapper(
+            "aworld.models.llm",
+            "LLMModel.stream_completion",
+            _stream_completion_class_wrapper(tracer=tracer)
+        )
+        wrapt.wrap_function_wrapper(
+            "aworld.models.llm",
+            "LLMModel.acompletion",
+            _acompletion_class_wrapper(tracer)
+        )
+
+        wrapt.wrap_function_wrapper(
+            "aworld.models.llm",
+            "LLMModel.astream_completion",
+            _stream_completion_class_wrapper(tracer)
+        )
+        logger.info(f"LLMModelInstrumentor wrap aworld.models.llm")
+
+    def _uninstrument(self, **kwargs: Any):
+        pass
+
+
+def wrap_llmmodel(client: 'aworld.models.llm.LLMModel'):
+    try:
+        tracer_provider = get_tracer_provider_silent()
+        if not tracer_provider:
+            return client
+        tracer = tracer_provider.get_tracer(
+            "aworld.trace.instrumentation.llmmodel")
+
+        wrapper = _completion_instance_wrapper(tracer)
+        awrapper = _acompletion_instance_wrapper(tracer)
+        stream_wrapper = _stream_completion_instance_wrapper(tracer)
+        client.completion = wrapper(client.completion)
+        client.stream_completion = stream_wrapper(client.stream_completion)
+        client.acompletion = awrapper(client.acompletion)
+        client.astream_completion = stream_wrapper(client.astream_completion)
+    except Exception:
+        logger.warning(traceback.format_exc())
+
+    return client

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/uni_llmmodel/model_response_parse.py

@@ -0,0 +1,183 @@
+import copy
+import json
+import aworld.trace.instrumentation.semconv as semconv
+from aworld.models.model_response import ModelResponse, ToolCall
+from aworld.trace.base import Span
+from aworld.trace.instrumentation.openai.inout_parse import should_trace_prompts, need_flatten_messages
+from aworld.logs.util import logger
+from aworld.utils.serialized_util import to_serializable
+
+
+def parser_request_params(kwargs, instance: 'aworld.models.llm.LLMModel'):
+    attributes = {
+        semconv.GEN_AI_SYSTEM: instance.provider_name,
+        semconv.GEN_AI_REQUEST_MODEL: instance.provider.model_name,
+        semconv.GEN_AI_REQUEST_MAX_TOKENS: kwargs.get("max_tokens", ""),
+        semconv.GEN_AI_REQUEST_TEMPERATURE: kwargs.get("temperature", ""),
+        semconv.GEN_AI_REQUEST_STOP_SEQUENCES: str(kwargs.get("stop", [])),
+        semconv.GEN_AI_REQUEST_FREQUENCY_PENALTY: kwargs.get("frequency_penalty", ""),
+        semconv.GEN_AI_REQUEST_PRESENCE_PENALTY: kwargs.get("presence_penalty", ""),
+        semconv.GEN_AI_REQUEST_USER: kwargs.get("user", ""),
+        semconv.GEN_AI_REQUEST_EXTRA_HEADERS: kwargs.get("extra_headers", ""),
+        semconv.GEN_AI_REQUEST_STREAMING: kwargs.get("stream", ""),
+        semconv.GEN_AI_REQUEST_TOP_P: kwargs.get("top_p", ""),
+        semconv.GEN_AI_OPERATION_NAME: "chat"
+    }
+    return attributes
+
+
+async def handle_request(span: Span, kwargs, instance):
+    if not span or not span.is_recording():
+        return
+    try:
+        attributes = parser_request_params(kwargs, instance)
+        if should_trace_prompts():
+            messages = kwargs.get("messages")
+            if need_flatten_messages():
+                attributes.update(parse_request_message(messages))
+            else:
+                attributes.update({
+                    semconv.GEN_AI_PROMPT: covert_to_jsonstr(messages)
+                })
+        tools = kwargs.get("tools")
+        if tools:
+            if need_flatten_messages():
+                attributes.update(parse_prompt_tools(tools))
+            else:
+                attributes.update({
+                    semconv.GEN_AI_PROMPT_TOOLS: covert_to_jsonstr(tools)
+                })
+
+        filterd_attri = {k: v for k, v in attributes.items()
+                         if (v and v != "")}
+
+        span.set_attributes(filterd_attri)
+    except Exception as e:
+        logger.warning(f"trace handle openai request error: {e}")
+
+
+def get_common_attributes_from_response(instance: 'LLMModel', is_async, is_streaming):
+    operation = "acompletion" if is_async else "completion"
+    if is_streaming:
+        operation = "astream_completion" if is_async else "stream_completion"
+    return {
+        semconv.GEN_AI_SYSTEM: instance.provider_name,
+        semconv.GEN_AI_RESPONSE_MODEL: instance.provider.model_name,
+        semconv.GEN_AI_METHOD_NAME: operation,
+        semconv.GEN_AI_SERVER_ADDRESS: instance.provider.base_url
+    }
+
+
+def accumulate_stream_response(chunk: ModelResponse, complete_response: dict):
+    from aworld.utils.common import nest_dict_counter
+    # logger.info(f"accumulate_stream_response chunk= {chunk}")
+
+    complete_response["model"] = chunk.model
+    complete_response["id"] = chunk.id
+    if chunk.content:
+        complete_response["content"] += chunk.content
+    if chunk.tool_calls:
+        complete_response["tool_calls"].extend(chunk.tool_calls)
+    if chunk.error:
+        complete_response["error"] = chunk.error
+    complete_response["usage"] = nest_dict_counter(
+        complete_response["usage"], chunk.usage)
+
+
+def record_stream_token_usage(complete_response, request_kwargs) -> tuple[int, int]:
+    '''
+        return (prompt_usage, completion_usage)
+    '''
+    # logger.info(
+    #     f"record_stream_token_usage complete_response= {complete_response}")
+    usage = complete_response.get("usage", {})
+    if usage:
+        prompt_tokens = usage.get("prompt_tokens", 0)
+        completion_tokens = usage.get("completion_tokens", 0)
+        return (prompt_tokens, completion_tokens)
+    return (0, 0)
+
+
+def parse_request_message(messages):
+    '''
+    flatten request message to attributes
+    '''
+    attributes = {}
+    for i, msg in enumerate(messages):
+        prefix = f"{semconv.GEN_AI_PROMPT}.{i}"
+        attributes.update({f"{prefix}.role": msg.get("role")})
+        if msg.get("content"):
+            content = copy.deepcopy(msg.get("content"))
+            content = json.dumps(content, ensure_ascii=False)
+            attributes.update({f"{prefix}.content": content})
+        if msg.get("tool_call_id"):
+            attributes.update({
+                f"{prefix}.tool_call_id": msg.get("tool_call_id")})
+        tool_calls = msg.get("tool_calls")
+        # logger.info(f"input tool_calls={tool_calls}")
+        if tool_calls:
+            for i, tool_call in enumerate(tool_calls):
+                if isinstance(tool_call, dict):
+                    function = tool_call.get('function')
+                    attributes.update({
+                        f"{prefix}.tool_calls.{i}.id": tool_call.get("id")})
+                    attributes.update({
+                        f"{prefix}.tool_calls.{i}.name": function.get("name")})
+                    attributes.update({
+                        f"{prefix}.tool_calls.{i}.arguments": function.get("arguments")})
+                elif isinstance(tool_call, ToolCall):
+                    function = tool_call.function
+                    attributes.update({
+                        f"{prefix}.tool_calls.{i}.id": tool_call.id})
+                    attributes.update({
+                        f"{prefix}.tool_calls.{i}.name": function.name})
+                    attributes.update({
+                        f"{prefix}.tool_calls.{i}.arguments": function.arguments})
+    return attributes
+
+
+def parse_prompt_tools(tools):
+    attributes = {}
+    for i, tool in enumerate(tools):
+        prefix = f"{semconv.GEN_AI_PROMPT_TOOLS}.{i}"
+        if isinstance(tool, dict):
+            tool_type = tool.get("type")
+            attributes.update({
+                f"{prefix}.type": tool_type})
+            if tool.get(tool_type):
+                attributes.update({
+                    f"{prefix}.name": tool.get(tool_type).get("name")})
+    return attributes
+
+
+def parse_response_message(tool_calls) -> dict:
+    attributes = {}
+    prefix = semconv.GEN_AI_COMPLETION_TOOL_CALLS
+    if tool_calls:
+        if need_flatten_messages():
+            for i, tool_call in enumerate(tool_calls):
+                function = tool_call.get("function")
+                attributes.update(
+                    {f"{prefix}.{i}.id": tool_call.get("id")})
+                attributes.update(
+                    {f"{prefix}.{i}.name": function.get("name")})
+                attributes.update(
+                    {f"{prefix}.{i}.arguments": function.get("arguments")})
+        else:
+            attributes.update({
+                prefix: covert_to_jsonstr(tool_calls)
+            })
+    return attributes
+
+
+def response_to_dic(response: ModelResponse) -> dict:
+    logger.info(f"completion response= {response}")
+    return response.to_dict()
+
+
+def covert_to_jsonstr(obj):
+    try:
+        return json.dumps(to_serializable(obj), ensure_ascii=False)
+    except:
+        logger.warning(f"covert_to_jsonstr error: {obj.__class__.__name__}")
+        return str(obj)

chapter9/gaia-experience/AWorld/aworld/trace/instrumentation/utils.py

@@ -0,0 +1,31 @@
+from importlib import import_module
+from wrapt import ObjectProxy
+
+
+def unwrap(obj: object, attr: str):
+    """Given a function that was wrapped by wrapt.wrap_function_wrapper, unwrap it
+
+    The object containing the function to unwrap may be passed as dotted module path string.
+
+    Args:
+        obj: Object that holds a reference to the wrapped function or dotted import path as string
+        attr (str): Name of the wrapped function
+    """
+    if isinstance(obj, str):
+        try:
+            module_path, class_name = obj.rsplit(".", 1)
+        except ValueError as exc:
+            raise ImportError(
+                f"Cannot parse '{obj}' as dotted import path"
+            ) from exc
+        module = import_module(module_path)
+        try:
+            obj = getattr(module, class_name)
+        except AttributeError as exc:
+            raise ImportError(
+                f"Cannot import '{class_name}' from '{module}'"
+            ) from exc
+
+    func = getattr(obj, attr, None)
+    if func and isinstance(func, ObjectProxy) and hasattr(func, "__wrapped__"):
+        setattr(obj, attr, func.__wrapped__)

chapter9/gaia-experience/AWorld/aworld/trace/msg_format.py

@@ -0,0 +1,403 @@
+import ast
+import inspect
+import sys
+import types
+import warnings
+import executing
+from functools import lru_cache
+from string import Formatter
+from types import CodeType
+from typing import Any, Literal, TypeVar
+from typing_extensions import NotRequired, TypedDict
+from .constants import MESSAGE_FORMATTED_VALUE_LENGTH_LIMIT
+from .stack_info import get_user_frame_and_stacklevel
+
+Truncatable = TypeVar('Truncatable', str, bytes, 'list[Any]', 'tuple[Any, ...]')
+
+class LiteralChunk(TypedDict):
+    t: Literal['lit']
+    v: str
+
+
+class ArgChunk(TypedDict):
+    t: Literal['arg']
+    v: str
+    spec: NotRequired[str]
+
+
+class KnownFormattingError(Exception):
+    """An error raised when there's something wrong with a format string or the field values.
+
+    In other words this should correspond to errors that would be raised when using `str.format`,
+    and generally indicate a user error, most likely that they weren't trying to pass a template string at all.
+    """
+
+
+class FStringAwaitError(Exception):
+    """An error raised when an await expression is found in an f-string.
+
+    This is a specific case that can't be handled by f-string introspection and requires
+    pre-evaluating the await expression before logging.
+    """
+
+
+class FormattingFailedWarning(UserWarning):
+    pass
+
+class InspectArgumentsFailedWarning(Warning):
+    pass
+
+class ChunksFormatter(Formatter):
+    def chunks(
+        self,
+        format_string: str,
+        kwargs: dict[str, Any],
+        *,
+        fstring_frame: types.FrameType = None,
+    ) -> tuple[list[LiteralChunk | ArgChunk], dict[str, Any], str]:
+        # Returns
+        # 1. A list of chunks
+        # 2. A dictionary of extra attributes to add to the span/log.
+        #      These can come from evaluating values in f-strings,
+        #      or from noting scrubbed values.
+        # 3. The final message template, which may differ from `format_string` if it was an f-string.
+        if fstring_frame:
+            result = self._fstring_chunks(kwargs, fstring_frame)
+            if result:  # returns None if faile
+                return result
+
+        chunks = self._vformat_chunks(
+            format_string,
+            kwargs=kwargs
+        )
+        # When there's no f-string magic, there's no changes in the template string.
+        return chunks, {}, format_string
+
+    def _fstring_chunks(
+        self,
+        kwargs: dict[str, Any],
+        frame: types.FrameType,
+    ) -> tuple[list[LiteralChunk | ArgChunk], dict[str, Any], str]:
+        # `frame` is the frame of the method that's being called by the user
+        # called_code = frame.f_code
+        frame = frame.f_back or frame  # type: ignore
+        assert frame is not None
+        # This is where the magic happens. It has caching.
+        ex = executing.Source.executing(frame)
+
+        call_node = ex.node
+        if call_node is None:  # type: ignore[reportUnnecessaryComparison]
+            # `executing` failed to find a node.
+            # This shouldn't happen in most cases, but it's best not to rely on it always working.
+            if not ex.source.text:
+                # This is a very likely cause.
+                # There's nothing we could possibly do to make magic work here,
+                # and it's a clear case where the user should turn the magic off.
+                warn_inspect_arguments(
+                    'No source code available. '
+                    'This happens when running in an interactive shell, '
+                    'using exec(), or running .pyc files without the source .py files.',
+                    get_stacklevel(frame),
+                )
+                return None
+
+            msg = '`executing` failed to find a node.'
+            if sys.version_info[:2] < (3, 11):  # pragma: no cover
+                # inspect_arguments is only on by default for 3.11+ for this reason.
+                # The AST modifications made by auto-tracing
+                # mean that the bytecode doesn't match the source code seen by `executing`.
+                # In 3.11+, a different algorithm is used by `executing` which can deal with this.
+                msg += ' This may be caused by a combination of using Python < 3.11 and auto-tracing.'
+
+            # Try a simple fallback heuristic to find the node which should work in most cases.
+            main_nodes: list[ast.AST] = []
+            for statement in ex.statements:
+                if isinstance(statement, ast.With):
+                    # Only look at the 'header' of a with statement, not its body.
+                    main_nodes += statement.items
+                else:
+                    main_nodes.append(statement)
+            call_nodes = [
+                node
+                for main_node in main_nodes
+                for node in ast.walk(main_node)
+                if isinstance(node, ast.Call)
+                if node.args or node.keywords
+            ]
+            if len(call_nodes) != 1:
+                warn_inspect_arguments(msg, get_stacklevel(frame))
+                return None
+
+            [call_node] = call_nodes
+
+        if not isinstance(call_node, ast.Call):  # pragma: no cover
+            # Very unlikely.
+            warn_inspect_arguments(
+                '`executing` unexpectedly identified a non-Call node.',
+                get_stacklevel(frame),
+            )
+            return None
+        
+        if call_node.args:
+            arg_node = call_node.args[0]
+        else:
+            # Very unlikely.
+            warn_inspect_arguments(
+                "Couldn't identify the `msg_template` argument in the call.",
+                get_stacklevel(frame),
+            )
+            return None
+
+        if not isinstance(arg_node, ast.JoinedStr):
+            # Not an f-string, not a problem.
+            # Just use normal formatting.
+            return None
+
+        # We have an f-string AST node.
+        # Now prepare the namespaces that we will use to evaluate the components.
+        global_vars = frame.f_globals
+        local_vars = {**frame.f_locals, **kwargs}
+
+        # Now for the actual formatting!
+        result: list[LiteralChunk | ArgChunk] = []
+
+        # We construct the message template (i.e. the span name) from the AST.
+        # We don't use the source code of the f-string because that gets messy
+        # if there's escaped quotes or implicit joining of adjacent strings.
+        new_template = ''
+
+        extra_attrs: dict[str, Any] = {}
+        for node_value in arg_node.values:
+            if isinstance(node_value, ast.Constant):
+                # These are the parts of the f-string not enclosed by `{}`, e.g. 'foo ' in f'foo {bar}'
+                value: str = node_value.value
+                result.append({'v': value, 't': 'lit'})
+                new_template += value
+            else:
+                # These are the parts of the f-string enclosed by `{}`, e.g. 'bar' in f'foo {bar}'
+                assert isinstance(node_value, ast.FormattedValue)
+
+                # This is cached.
+                source, value_code, formatted_code = compile_formatted_value(node_value, ex.source)
+
+                # Note that this doesn't include:
+                # - The format spec, e.g. `:0.2f`
+                # - The conversion, e.g. `!r`
+                # - The '=' sign within the braces, e.g. `{bar=}`.
+                #     The AST represents f'{bar = }' as f'bar = {bar}' which is how the template will look.
+                new_template += '{' + source + '}'
+
+                # The actual value of the expression.
+                value = eval(value_code, global_vars, local_vars)
+                extra_attrs[source] = value
+
+                # Format the value according to the format spec, converting to a string.
+                formatted = eval(formatted_code, global_vars, {**local_vars, '@fvalue': value})
+                formatted = self._clean_value(formatted)
+                result.append({'v': formatted, 't': 'arg'})
+
+        return result, extra_attrs, new_template
+
+    def _vformat_chunks(
+        self,
+        format_string: str,
+        kwargs: dict[str, Any],
+        *,
+        recursion_depth: int = 2,
+    ) -> list[LiteralChunk | ArgChunk]:
+        """Copied from `string.Formatter._vformat` https://github.com/python/cpython/blob/v3.11.4/Lib/string.py#L198-L247 then altered."""
+        if recursion_depth < 0:
+            raise KnownFormattingError('Max format spec recursion exceeded')
+        result: list[LiteralChunk | ArgChunk] = []
+        # We currently don't use positional arguments
+        args = ()
+
+        for literal_text, field_name, format_spec, conversion in self.parse(format_string):
+            # output the literal text
+            if literal_text:
+                result.append({'v': literal_text, 't': 'lit'})
+
+            # if there's a field, output it
+            if field_name is not None:
+                # this is some markup, find the object and do
+                #  the formatting
+                if field_name == '':
+                    raise KnownFormattingError('Empty curly brackets `{}` are not allowed. A field name is required.')
+
+                # ADDED BY US:
+                if field_name.endswith('='):
+                    if result and result[-1]['t'] == 'lit':
+                        result[-1]['v'] += field_name
+                    else:
+                        result.append({'v': field_name, 't': 'lit'})
+                    field_name = field_name[:-1]
+
+                # given the field_name, find the object it references
+                #  and the argument it came from
+                try:
+                    obj, _arg_used = self.get_field(field_name, args, kwargs)
+                except IndexError:
+                    raise KnownFormattingError('Numeric field names are not allowed.')
+                except KeyError as exc1:
+                    if str(exc1) == repr(field_name):
+                        raise KnownFormattingError(f'The field {{{field_name}}} is not defined.') from exc1
+
+                    try:
+                        # field_name is something like 'a.b' or 'a[b]'
+                        # Evaluating that expression failed, so now just try getting the whole thing from kwargs.
+                        # In particular, OTEL attributes with dots in their names are normal and handled here.
+                        obj = kwargs[field_name]
+                    except KeyError as exc2:
+                        # e.g. neither 'a' nor 'a.b' is defined
+                        raise KnownFormattingError(f'The fields {exc1} and {exc2} are not defined.') from exc2
+                except Exception as exc:
+                    raise KnownFormattingError(f'Error getting field {{{field_name}}}: {exc}') from exc
+
+                # do any conversion on the resulting object
+                if conversion is not None:
+                    try:
+                        obj = self.convert_field(obj, conversion)
+                    except Exception as exc:
+                        raise KnownFormattingError(f'Error converting field {{{field_name}}}: {exc}') from exc
+
+                # expand the format spec, if needed
+                format_spec_chunks = self._vformat_chunks(
+                    format_spec or '', kwargs, recursion_depth=recursion_depth - 1
+                )
+                format_spec = ''.join(chunk['v'] for chunk in format_spec_chunks)
+
+                try:
+                    value = self.format_field(obj, format_spec)
+                except Exception as exc:
+                    raise KnownFormattingError(f'Error formatting field {{{field_name}}}: {exc}') from exc
+                value = self._clean_value(value)
+                d: ArgChunk = {'v': value, 't': 'arg'}
+                if format_spec:
+                    d['spec'] = format_spec
+                result.append(d)
+
+        return result
+
+    def _clean_value(self, value: str) -> str:
+        return truncate_sequence(seq=value, max_length=MESSAGE_FORMATTED_VALUE_LENGTH_LIMIT, middle='...')
+
+def warn_inspect_arguments(msg: str, stacklevel: int):
+    """Warn about an error in inspecting arguments.
+    This is a separate function so that it can be called from multiple places.
+    """
+    msg = (
+        'Failed to introspect calling code. '
+        'Falling back to normal message formatting '
+        'which may result in loss of information if using an f-string. '
+        'The problem was:\n'
+    ) + msg
+    warnings.warn(msg, InspectArgumentsFailedWarning, stacklevel=stacklevel)
+
+
+def get_stacklevel(frame: types.FrameType):
+    """Get a stacklevel which can be passed to warn_inspect_arguments
+    which points at the given frame, where the f-string was found.
+    """
+    current_frame = inspect.currentframe()
+    stacklevel = 0
+    while current_frame:  # pragma: no branch
+        if current_frame == frame:
+            break
+        stacklevel += 1
+        current_frame = current_frame.f_back
+    return stacklevel
+
+@lru_cache
+def compile_formatted_value(node: ast.FormattedValue, ex_source: executing.Source) -> tuple[str, CodeType, CodeType]:
+    """Returns three things that can be expensive to compute.
+
+    1. Source code corresponding to the node value (excluding the format spec).
+    2. A compiled code object which can be evaluated to calculate the value.
+    3. Another code object which formats the value.
+    """
+    source = get_node_source_text(node.value, ex_source)
+
+    # Check if the expression contains await before attempting to compile
+    for sub_node in ast.walk(node.value):
+        if isinstance(sub_node, ast.Await):
+            raise FStringAwaitError(source)
+
+    value_code = compile(source, '<fvalue1>', 'eval')
+    expr = ast.Expression(
+        ast.JoinedStr(
+            values=[
+                # Similar to the original FormattedValue node,
+                # but replace the actual expression with a simple variable lookup
+                # so that it the expression doesn't need to be evaluated again.
+                # Use @ in the variable name so that it can't possibly conflict
+                # with a normal variable.
+                # The value of this variable will be provided in the eval() call
+                # and will come from evaluating value_code above.
+                ast.FormattedValue(
+                    value=ast.Name(id='@fvalue', ctx=ast.Load()),
+                    conversion=node.conversion,
+                    format_spec=node.format_spec,
+                )
+            ]
+        )
+    )
+    ast.fix_missing_locations(expr)
+    formatted_code = compile(expr, '<fvalue2>', 'eval')
+    return source, value_code, formatted_code
+
+def get_node_source_text(node: ast.AST, ex_source: executing.Source):
+    """Returns some Python source code representing `node`.
+
+    Preferably the actual original code given by `ast.get_source_segment`,
+    but falling back to `ast.unparse(node)` if the former is incorrect.
+    This happens sometimes due to Python bugs (especially for older Python versions)
+    in the source positions of AST nodes inside f-strings.
+    """
+    # ast.unparse is not available in Python 3.8, which is why inspect_arguments is forbidden in 3.8.
+    source_unparsed = ast.unparse(node)
+    source_segment = ast.get_source_segment(ex_source.text, node) or ''
+    try:
+        # Verify that the source segment is correct by checking that the AST is equivalent to what we have.
+        source_segment_unparsed = ast.unparse(ast.parse(source_segment, mode='eval'))
+    except Exception:  # probably SyntaxError, but ast.parse can raise other exceptions too
+        source_segment_unparsed = ''
+    return source_segment if source_unparsed == source_segment_unparsed else source_unparsed
+
+
+def truncate_sequence(seq: Truncatable, *, max_length: int, middle: Truncatable) -> Truncatable:
+    """Return a sequence at with `len()` at most `max_length`, with `middle` in the middle if truncated."""
+    if len(seq) <= max_length:
+        return seq
+    remaining_length = max_length - len(middle)
+    half = remaining_length // 2
+    return seq[:half] + middle + seq[-half:]
+
+def warn_at_user_stacklevel(msg: str, category: type[Warning]):
+    """Warn at the user's stack level.
+    """
+    _frame, stacklevel = get_user_frame_and_stacklevel()
+    warnings.warn(msg, stacklevel=stacklevel, category=category)
+
+def warn_formatting(msg: str):
+    """Warn about a formatting error.   
+    """
+    warn_at_user_stacklevel(
+        f'\n'
+        f'    Ensure you are either:\n'
+        '      (1) passing an f-string directly, or\n'
+        '      (2) passing a literal `str.format`-style template, not a preformatted string.\n'
+        f'    The problem was: {msg}',
+        category=FormattingFailedWarning,
+    )
+
+def warn_fstring_await(msg: str):
+    """Warn about an await expression in an f-string.
+    """
+    warn_at_user_stacklevel(
+        f'\n'
+        f'    Cannot evaluate await expression in f-string. Pre-evaluate the expression before logging.\n'
+        f'    The problematic f-string value was: {msg}',
+        category=FormattingFailedWarning,
+    )
+
+chunks_formatter = ChunksFormatter()

chapter9/gaia-experience/AWorld/aworld/trace/opentelemetry/__init__.py

@@ -0,0 +1,2 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.

chapter9/gaia-experience/AWorld/aworld/trace/opentelemetry/export/__init__.py

@@ -0,0 +1,135 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+import typing
+from os import linesep
+from aworld.trace.base import Span
+from aworld.trace.span_cosumer import SpanConsumer, get_span_consumers
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExportResult, SpanExporter
+from aworld.logs.util import logger
+
+
+class FileSpanExporter(SpanExporter):
+    """Implementation of :class:`SpanExporter` that prints spans to the
+    console.
+
+    This class can be used for diagnostic purposes. It prints the exported
+    spans to the console STDOUT.
+    """
+
+    def __init__(
+        self,
+        file_path: str = None,
+        formatter: typing.Callable[
+            [ReadableSpan], str
+        ] = lambda span: span.to_json() + linesep,
+    ):
+        self.formatter = formatter
+        self.file_path = file_path
+
+    def export(self, spans: typing.Sequence[ReadableSpan]) -> SpanExportResult:
+        try:
+            with open(self.file_path, 'a') as f:
+                for span in spans:
+                    f.write(self.formatter(span))
+
+            return SpanExportResult.SUCCESS
+        except Exception as e:
+            logger.error(e)
+            return SpanExportResult.FAILURE
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:
+        return True
+
+
+class ReadOnlySpan(Span, ReadableSpan):
+    """Implementation of :class:`Span` that wraps a :class:`ReadableSpan`.
+    This class can be used to wrap a :class:`ReadableSpan` to make it
+    read-only.
+    Args:
+        span: The span to wrap.
+    """
+
+    def __init__(self, span: ReadableSpan):
+        self._span = span
+
+    if not typing.TYPE_CHECKING:
+        def __getattr__(self, name: str) -> typing.Any:
+            return getattr(self._span, name)
+
+    def end(self, end_time: typing.Optional[int] = None) -> None:
+        pass
+
+    def set_attribute(self, key: str, value: typing.Any) -> None:
+        pass
+
+    def set_attributes(self, attributes: dict[str, typing.Any]) -> None:
+        pass
+
+    def is_recording(self) -> bool:
+        return False
+
+    def record_exception(
+            self,
+            exception: BaseException,
+            attributes: dict[str, typing.Any] = None,
+            timestamp: typing.Optional[int] = None,
+            escaped: bool = False,
+    ) -> None:
+        pass
+
+    def get_trace_id(self) -> str:
+        return f"{self._span.get_span_context().trace_id:032x}"
+
+    def get_span_id(self) -> str:
+        return f"{self._span.get_span_context().span_id:016x}"
+
+
+class SpanConsumerExporter(SpanExporter):
+    """Implementation of :class:`SpanExporter` that exports spans to
+    multiple span consumers.
+    This class can be used for exporting spans to multiple span consumers.
+    It exports the spans to the span consumers in the order they are passed
+    in the constructor.
+    Args:
+        span_consumers: A sequence of span consumers to export spans to. 
+    """
+
+    def __init__(
+        self,
+        span_consumers: typing.Sequence[SpanConsumer] = None,
+    ):
+        self._span_consumers = span_consumers or []
+        self._loaded = False
+
+    def _load_span_consumers(self):
+        if not self._loaded:
+            self._span_consumers.extend(get_span_consumers())
+            self._loaded = True
+
+    def export(
+        self, spans: typing.Sequence[ReadableSpan]
+    ) -> SpanExportResult:
+        self._load_span_consumers()
+        span_batches = []
+        for span in spans:
+            span_batches.append(ReadOnlySpan(span))
+        for span_consumer in self._span_consumers:
+            try:
+                span_consumer.consume(span_batches)
+            except Exception as e:
+                logger.error(
+                    f"Error consume spans: {e}, span_consumer: {span_consumer.__class__.__name__}")
+        return SpanExportResult.SUCCESS
+
+
+class NoOpSpanExporter(SpanExporter):
+    """Implementation of :class:`SpanExporter` that does not export spans."""
+
+    def export(
+        self, spans: typing.Sequence[ReadableSpan]
+    ) -> SpanExportResult:
+        return SpanExportResult.SUCCESS
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:
+        return True

chapter9/gaia-experience/AWorld/aworld/trace/opentelemetry/memory_storage.py

@@ -0,0 +1,238 @@
+import os
+import json
+import time
+import threading
+from datetime import datetime
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from pydantic import BaseModel
+from typing import Optional, Dict, Any, Union
+from opentelemetry.sdk.trace import Span, SpanContext
+from opentelemetry.sdk.trace.export import SpanExporter
+from aworld.logs.util import logger
+from aworld.trace.constants import ATTRIBUTES_MESSAGE_RUN_TYPE_KEY, RunType
+
+
+class SpanStatus(BaseModel):
+    code: str = "UNSET"
+    description: Optional[str] = None
+
+
+class SpanModel(BaseModel):
+    trace_id: str
+    span_id: str
+    name: str
+    start_time: str
+    end_time: str
+    duration_ms: float
+    attributes: Dict[str, Any]
+    status: SpanStatus
+    parent_id: Optional[str]
+    children: list['SpanModel'] = []
+    run_type: Optional[str] = RunType.OTHER.value
+    is_event: bool = False
+
+    @staticmethod
+    def from_span(span):
+        start_timestamp = span.start_time / 1e9
+        end_timestamp = span.end_time / 1e9
+        start_ms = int((span.start_time % 1e9) / 1e6)
+        end_ms = int((span.end_time % 1e9) / 1e6)
+
+        return SpanModel(
+            trace_id=f"{span.get_span_context().trace_id:032x}",
+            span_id=SpanModel.get_span_id(span),
+            name=span.name,
+            start_time=time.strftime(
+                '%Y-%m-%d %H:%M:%S', time.localtime(start_timestamp)) + f'.{start_ms:03d}',
+            end_time=time.strftime(
+                '%Y-%m-%d %H:%M:%S', time.localtime(end_timestamp)) + f'.{end_ms:03d}',
+            duration_ms=(span.end_time - span.start_time)/1e6,
+            attributes={k: v for k, v in span.attributes.items()},
+            status=SpanStatus(
+                code=str(
+                    span.status.status_code) if span.status.status_code else "UNSET",
+                description=span.status.description or None
+            ),
+            parent_id=SpanModel.get_span_id(
+                span.parent) if span.parent else None,
+            run_type=span.attributes.get(
+                ATTRIBUTES_MESSAGE_RUN_TYPE_KEY, RunType.OTHER.value),
+            is_event=(span.attributes.get("event.id") is not None)
+        )
+
+    @staticmethod
+    def get_span_id(span: Union[Span, SpanContext]):
+        if isinstance(span, SpanContext):
+            return f"{span.span_id:016x}"
+        return f"{span.get_span_context().span_id:016x}"
+
+
+class TraceStorage(ABC):
+    """
+    Storage for traces.
+    """
+    @abstractmethod
+    def add_span(self, span: Span) -> None:
+        """
+        Add a span to the storage.
+        """
+
+    @abstractmethod
+    def get_all_traces(self) -> list[str]:
+        """
+        Get all trace ids.
+        """
+
+    @abstractmethod
+    def get_all_spans(self, trace_id) -> list[SpanModel]:
+        """
+        Get all spans of a trace.
+        """
+
+
+class InMemoryStorage(TraceStorage):
+    """
+    In-memory storage for spans.
+    """
+
+    def __init__(self,  max_traces=1000):
+        self._traces = defaultdict(list)
+        self._trace_order = []
+        self.max_traces = max_traces
+
+    def add_span(self, span: Span):
+        trace_id = f"{span.get_span_context().trace_id:032x}"
+        if trace_id not in self._traces:
+            self._trace_order.append(trace_id)
+            if len(self._trace_order) > self.max_traces:
+                oldest_trace = self._trace_order.pop(0)
+                del self._traces[oldest_trace]
+        self._traces[trace_id].append(SpanModel.from_span(span))
+
+    def get_all_traces(self):
+        return list(self._traces.keys())
+
+    def get_all_spans(self, trace_id):
+        return self._traces.get(trace_id, [])
+
+
+class InMemoryWithPersistStorage(TraceStorage):
+    """
+    In-memory storage for spans with optimized disk persistence.
+    """
+
+    def __init__(self, storage_dir: str = "./trace_data"):
+        self._traces = defaultdict(list)
+        self._pending_spans = []
+        self.storage_dir = os.path.abspath(storage_dir)
+        os.makedirs(self.storage_dir, exist_ok=True)
+        self._lock = threading.Lock()
+        self._persist_thread = None
+        self._load_today_traces()
+        self.current_filename = None
+
+    def _get_today_filename(self):
+        if not self.current_filename:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            self.current_filename = f"trace_{timestamp}.json"
+        return self.current_filename
+
+    def _load_today_traces(self):
+        today = datetime.now().strftime("%Y%m%d")
+        for filename in os.listdir(self.storage_dir):
+            if filename.startswith(f"trace_{today}") and filename.endswith(".json"):
+                filepath = os.path.join(self.storage_dir, filename)
+                try:
+                    with self._lock, open(filepath, 'r') as f:
+                        data = json.load(f)
+                        for span_data in data:
+                            trace_id = span_data.get("trace_id")
+                            span_json = span_data.get("span")
+                            self._traces[trace_id].append(
+                                SpanModel.parse_raw(span_json))
+                except Exception as e:
+                    logger.error(
+                        f"Error loading trace file {filename}: {str(e)}")
+
+    def _start_persist_thread(self):
+        if self._persist_thread is None:
+            self._persist_thread = threading.Thread(
+                target=self._persist_worker, daemon=True)
+            self._persist_thread.start()
+
+    def _persist_worker(self):
+        while True:
+            time.sleep(5)
+            self._persist()
+
+    def _persist(self):
+        if not self._pending_spans:
+            return
+
+        temp_filepath = os.path.join(
+            self.storage_dir, f"temp_{time.time_ns()}.json")
+        final_filepath = os.path.join(
+            self.storage_dir, self._get_today_filename())
+
+        try:
+            spans_to_persist = []
+            with self._lock:
+                spans_to_persist = self._pending_spans.copy()
+                self._pending_spans.clear()
+
+            if spans_to_persist:
+                existing_data = []
+                if os.path.exists(final_filepath):
+                    try:
+                        with open(final_filepath, 'r') as f:
+                            existing_data = json.load(f)
+                    except Exception as e:
+                        logger.error(
+                            f"Error reading existing trace file: {str(e)}")
+
+                merged_spans = existing_data + spans_to_persist
+
+                with open(temp_filepath, 'w') as f:
+                    json.dump(merged_spans, f, default=str)
+                os.replace(temp_filepath, final_filepath)
+        except Exception as e:
+            logger.error(f"Error persisting traces: {str(e)}")
+            try:
+                os.unlink(temp_filepath)
+            except:
+                pass
+
+    def add_span(self, span: Span):
+        span_model = SpanModel.from_span(span)
+        with self._lock:
+            self._traces[span_model.trace_id].append(span_model)
+            self._pending_spans.append({
+                "trace_id": span_model.trace_id,
+                "span": span_model.json()
+            })
+        self._start_persist_thread()
+
+    def get_all_traces(self):
+        with self._lock:
+            return list(self._traces.keys())
+
+    def get_all_spans(self, trace_id):
+        with self._lock:
+            return self._traces.get(trace_id, [])
+
+
+class InMemorySpanExporter(SpanExporter):
+    """
+    Span exporter that stores spans in memory.
+    """
+
+    def __init__(self, storage: TraceStorage):
+        self._storage = storage
+
+    def export(self, spans):
+        for span in spans:
+            self._storage.add_span(span)
+
+    def shutdown(self):
+        pass

chapter9/gaia-experience/AWorld/aworld/trace/opentelemetry/opentelemetry_adapter.py

@@ -0,0 +1,436 @@
+import sys
+import os
+import traceback
+import time
+import datetime
+import requests
+from threading import Lock
+from typing import Any, Iterator, Sequence, Optional, TYPE_CHECKING
+from contextvars import Token
+from urllib.parse import urljoin
+import opentelemetry.context as otlp_context_api
+from opentelemetry.trace import (
+    SpanKind,
+    set_span_in_context,
+    get_current_span as get_current_otlp_span,
+    NonRecordingSpan,
+    SpanContext,
+    TraceFlags
+)
+from opentelemetry.trace.status import StatusCode
+from opentelemetry.sdk.trace import (
+    ReadableSpan,
+    SynchronousMultiSpanProcessor,
+    Tracer as SDKTracer,
+    Span as SDKSpan,
+    TracerProvider as SDKTracerProvider
+)
+from opentelemetry.context import Context as OTLPContext
+from opentelemetry.semconv.trace import SpanAttributes
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor
+
+from aworld.trace.base import (
+    AttributeValueType,
+    NoOpTracer,
+    SpanType,
+    TraceProvider,
+    Tracer,
+    Span,
+    TraceContext,
+    set_tracer_provider
+)
+from aworld.trace.span_cosumer import SpanConsumer
+from aworld.trace.propagator import get_global_trace_context
+from aworld.trace.baggage.sofa_tracer import SofaSpanHelper
+from aworld.logs.util import logger
+from aworld.utils.common import get_local_ip
+from .memory_storage import InMemorySpanExporter, InMemoryStorage
+from ..constants import ATTRIBUTES_MESSAGE_KEY
+from .export import FileSpanExporter, NoOpSpanExporter, SpanConsumerExporter
+from ..server import set_trace_server
+
+
+class OTLPTraceProvider(TraceProvider):
+    """A TraceProvider that wraps an existing `SDKTracerProvider`.
+    This class provides a way to use a `SDKTracerProvider` as a `TraceProvider`.
+    When the context manager is entered, it returns the `SDKTracerProvider` itself.
+    When the context manager is exited, it calls `shutdown` on the `SDKTracerProvider`.
+    Args:
+        provider: The internal provider to wrap.
+    """
+
+    def __init__(self, provider: SDKTracerProvider, suppressed_scopes: Optional[set[str]] = None):
+        self._provider: SDKTracerProvider = provider
+        self._suppressed_scopes = set()
+        if suppressed_scopes:
+            self._suppressed_scopes.update(suppressed_scopes)
+        self._lock: Lock = Lock()
+
+    def get_tracer(
+            self,
+            name: str,
+            version: Optional[str] = None
+    ):
+        with self._lock:
+            if name in self._suppressed_scopes:
+                return NoOpTracer()
+            else:
+                tracer = self._provider.get_tracer(instrumenting_module_name=name,
+                                                   instrumenting_library_version=version)
+                return OTLPTracer(tracer)
+
+    def shutdown(self) -> None:
+        with self._lock:
+            if isinstance(self._provider, SDKTracerProvider):
+                self._provider.shutdown()
+
+    def force_flush(self, timeout: Optional[float] = None) -> bool:
+        with self._lock:
+            if isinstance(self._provider, SDKTracerProvider):
+                return self._provider.force_flush(timeout)
+            else:
+                return False
+
+    def get_current_span(self) -> Optional["Span"]:
+        otlp_span = get_current_otlp_span()
+        return OTLPSpan(otlp_span, is_new_span=False)
+
+
+class OTLPTracer(Tracer):
+    """A Tracer represents a collection of Spans.
+    Args:
+        tracer: The internal tracer to wrap.
+    """
+
+    def __init__(self, tracer: SDKTracer):
+        self._tracer = tracer
+
+    def start_span(
+            self,
+            name: str,
+            span_type: SpanType = SpanType.INTERNAL,
+            attributes: dict[str, AttributeValueType] = None,
+            start_time: Optional[int] = None,
+            record_exception: bool = True,
+            set_status_on_exception: bool = True,
+            trace_context: Optional[TraceContext] = None
+    ) -> "Span":
+        otel_context = None
+        trace_context = trace_context or get_global_trace_context().get_and_clear()
+        if trace_context:
+            otel_context = self._get_otel_context_from_trace_context(
+                trace_context)
+        start_time = start_time or time.time_ns()
+        attributes = {**(attributes or {})}
+        attributes.setdefault(ATTRIBUTES_MESSAGE_KEY, name)
+        SofaSpanHelper.set_sofa_context_to_attr(attributes)
+        attributes = {k: v for k, v in attributes.items(
+        ) if is_valid_attribute_value(k, v)}
+
+        span_kind = self._convert_to_span_kind(
+            span_type) if span_type else SpanKind.INTERNAL
+        span = self._tracer.start_span(name=name,
+                                       kind=span_kind,
+                                       context=otel_context,
+                                       attributes=attributes,
+                                       start_time=start_time,
+                                       record_exception=record_exception,
+                                       set_status_on_exception=set_status_on_exception)
+        return OTLPSpan(span)
+
+    def start_as_current_span(
+            self,
+            name: str,
+            span_type: SpanType = SpanType.INTERNAL,
+            attributes: dict[str, AttributeValueType] = None,
+            start_time: Optional[int] = None,
+            record_exception: bool = True,
+            set_status_on_exception: bool = True,
+            end_on_exit: bool = True,
+            trace_context: Optional[TraceContext] = None
+    ) -> Iterator["Span"]:
+
+        start_time = start_time or time.time_ns()
+        attributes = {**(attributes or {})}
+        attributes.setdefault(ATTRIBUTES_MESSAGE_KEY, name)
+        SofaSpanHelper.set_sofa_context_to_attr(attributes)
+        attributes = {k: v for k, v in attributes.items(
+        ) if is_valid_attribute_value(k, v)}
+
+        span_kind = self._convert_to_span_kind(
+            span_type) if span_type else SpanKind.INTERNAL
+        otel_context = None
+        trace_context = trace_context or get_global_trace_context().get_and_clear()
+        if trace_context:
+            otel_context = self._get_otel_context_from_trace_context(
+                trace_context)
+
+        class _OTLPSpanContextManager:
+            def __init__(self, tracer: SDKTracer):
+                self._span_cm = None
+                self._tracer = tracer
+
+            def __enter__(self):
+                self._span_cm = self._tracer.start_as_current_span(
+                    name=name,
+                    kind=span_kind,
+                    context=otel_context,
+                    attributes=attributes,
+                    start_time=start_time,
+                    record_exception=record_exception,
+                    set_status_on_exception=set_status_on_exception,
+                    end_on_exit=end_on_exit
+                )
+                inner_span = self._span_cm.__enter__()
+                return OTLPSpan(inner_span)
+
+            def __exit__(self, exc_type, exc_val, exc_tb):
+                return self._span_cm.__exit__(exc_type, exc_val, exc_tb)
+
+        return _OTLPSpanContextManager(self._tracer)
+
+    def _convert_to_span_kind(self, span_type: SpanType) -> str:
+        if span_type == SpanType.INTERNAL:
+            return SpanKind.INTERNAL
+        elif span_type == SpanType.CLIENT:
+            return SpanKind.CLIENT
+        elif span_type == SpanType.SERVER:
+            return SpanKind.SERVER
+        elif span_type == SpanType.PRODUCER:
+            return SpanKind.PRODUCER
+        elif span_type == SpanType.CONSUMER:
+            return SpanKind.CONSUMER
+        else:
+            return SpanKind.INTERNAL
+
+    def _get_otel_context_from_trace_context(self, trace_context: TraceContext) -> OTLPContext:
+        trace_flags = None
+        if trace_context.trace_flags:
+            trace_flags = TraceFlags(int(trace_context.trace_flags, 16))
+        otel_context = otlp_context_api.Context()
+        return set_span_in_context(
+            NonRecordingSpan(
+                SpanContext(
+                    trace_id=int(trace_context.trace_id, 16),
+                    span_id=int(trace_context.span_id, 16),
+                    is_remote=True,
+                    trace_flags=trace_flags
+                )
+            ),
+            otel_context,
+        )
+
+
+class OTLPSpan(Span, ReadableSpan):
+    """A Span represents a single operation within a trace.
+    """
+
+    def __init__(self, span: SDKSpan, is_new_span=True):
+        self._span = span
+        self._token: Optional[Token[OTLPContext]] = None
+        if is_new_span:
+            self._attach()
+            self._add_to_open_spans()
+
+    if not TYPE_CHECKING:  # pragma: no branch
+        def __getattr__(self, name: str) -> Any:
+            return getattr(self._span, name)
+
+    def end(self, end_time: Optional[int] = None) -> None:
+        self._remove_from_open_spans()
+        end_time = end_time or time.time_ns()
+        if not self._span._status or self._span._status.status_code == StatusCode.UNSET:
+            self._span.set_status(
+                status=StatusCode.OK,
+                description="",
+            )
+        self._span.end(end_time=end_time)
+        self._detach()
+
+    def set_attribute(self, key: str, value: Any) -> None:
+        if not is_valid_attribute_value(key, value):
+            return
+        self._span.set_attribute(key=key, value=value)
+
+    def set_attributes(self, attributes: dict[str, Any]) -> None:
+        attributes = {k: v for k, v in attributes.items(
+        ) if is_valid_attribute_value(k, v)}
+        self._span.set_attributes(attributes=attributes)
+
+    def is_recording(self) -> bool:
+        return self._span.is_recording()
+
+    def record_exception(
+            self,
+            exception: BaseException,
+            attributes: dict[str, Any] = None,
+            timestamp: Optional[int] = None,
+            escaped: bool = False,
+    ) -> None:
+        timestamp = timestamp or time.time_ns()
+        attributes = {**(attributes or {})}
+
+        stacktrace = ''.join(traceback.format_exception(
+            type(exception), exception, exception.__traceback__))
+        self._span.set_attributes({
+            SpanAttributes.EXCEPTION_STACKTRACE: stacktrace,
+            SpanAttributes.EXCEPTION_TYPE: type(exception).__name__,
+            SpanAttributes.EXCEPTION_MESSAGE: str(exception),
+            SpanAttributes.EXCEPTION_ESCAPED: escaped
+        })
+        if exception is not sys.exc_info()[1]:
+            attributes[SpanAttributes.EXCEPTION_STACKTRACE] = stacktrace
+
+        self._span.record_exception(exception=exception,
+                                    attributes=attributes,
+                                    timestamp=timestamp,
+                                    escaped=escaped)
+        self._span.set_status(
+            status=StatusCode.ERROR,
+            description=str(exception),
+        )
+
+    def get_trace_id(self) -> str:
+        """Get the trace ID of the span.
+        Returns:
+            The trace ID of the span.
+        """
+        if not self._span or not self._span.get_span_context() or not self.is_recording():
+            return None
+        return f"{self._span.get_span_context().trace_id:032x}"
+
+    def get_span_id(self) -> str:
+        """Get the span ID of the span.
+        Returns:
+            The span ID of the span.
+        """
+        if not self._span or not self._span.get_span_context() or not self.is_recording():
+            return None
+        return f"{self._span.get_span_context().span_id:016x}"
+
+    def _attach(self):
+        if self._token is not None:
+            return
+        self._token = otlp_context_api.attach(set_span_in_context(self._span))
+
+    def _detach(self):
+        if self._token is None:
+            return
+        try:
+            otlp_context_api.detach(self._token)
+        except ValueError as e:
+            logger.warning(f"Failed to detach context: {e}")
+        finally:
+            self._token = None
+
+
+def configure_otlp_provider(
+        backends: Sequence[str] = None,
+        base_url: str = None,
+        write_token: str = None,
+        span_consumers: Optional[Sequence[SpanConsumer]] = None,
+        **kwargs
+) -> None:
+    """Configure the OTLP provider.
+    Args:
+        backend: The backend to use.
+        write_token: The write token to use.
+        **kwargs: Additional keyword arguments to pass to the provider.
+    """
+    from aworld.metrics.opentelemetry.opentelemetry_adapter import build_otel_resource
+    backends = backends or ["logfire"]
+    processor = SynchronousMultiSpanProcessor()
+    processor.add_span_processor(BatchSpanProcessor(
+        SpanConsumerExporter(span_consumers)))
+    for backend in backends:
+        if backend == "logfire":
+            span_exporter = _configure_logfire_exporter(
+                write_token=write_token, base_url=base_url, **kwargs)
+            processor.add_span_processor(BatchSpanProcessor(span_exporter))
+        elif backend == "console":
+            from opentelemetry.sdk.trace.export import ConsoleSpanExporter
+            processor.add_span_processor(
+                BatchSpanProcessor(ConsoleSpanExporter()))
+        elif backend == "file":
+            timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+            file_path = kwargs.get("file_path", f"traces_{timestamp}.json")
+            processor.add_span_processor(
+                BatchSpanProcessor(FileSpanExporter(file_path)))
+        elif backend == "memory":
+            logger.info("Using in-memory storage for traces.")
+            storage = kwargs.get(
+                "storage", InMemoryStorage()) or InMemoryStorage()
+            processor.add_span_processor(
+                SimpleSpanProcessor(InMemorySpanExporter(storage=storage)))
+            server_enabled = str(kwargs.get("server_enabled")) or os.getenv(
+                "START_TRACE_SERVER") or "true"
+            server_port = kwargs.get("server_port") or 7079
+            if (server_enabled.lower() == "true"):
+                logger.info(f"Starting trace server on port {server_port}.")
+                set_trace_server(storage=storage, port=int(
+                    server_port), start_server=True)
+            else:
+                logger.info("Trace server is not started.")
+                set_trace_server(storage=storage, port=int(
+                    server_port), start_server=False)
+        else:
+            span_exporter = _configure_otlp_exporter(
+                base_url=base_url, **kwargs)
+            processor.add_span_processor(BatchSpanProcessor(span_exporter))
+
+    set_tracer_provider(OTLPTraceProvider(SDKTracerProvider(active_span_processor=processor,
+                                                            resource=build_otel_resource())))
+
+
+def _configure_logfire_exporter(write_token: str, base_url: str = None) -> None:
+    """Configure the Logfire exporter.
+    Args:
+        write_token: The write token to use.
+        base_url: The base URL to use.
+        **kwargs: Additional keyword arguments to pass to the exporter.
+    """
+    from opentelemetry.exporter.otlp.proto.http import Compression
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+
+    base_url = base_url or "https://logfire-us.pydantic.dev"
+    headers = {'User-Agent': f'logfire/3.14.0', 'Authorization': write_token}
+    session = requests.Session()
+    session.headers.update(headers)
+    return OTLPSpanExporter(
+        endpoint=urljoin(base_url, '/v1/traces'),
+        session=session,
+        compression=Compression.Gzip,
+    )
+
+
+def _configure_otlp_exporter(base_url: str = None, **kwargs) -> None:
+    """Configure the OTLP exporter.
+    Args:
+        write_token: The write token to use.
+        base_url: The base URL to use.
+        **kwargs: Additional keyword arguments to pass to the exporter.
+    """
+    import requests
+    from opentelemetry.exporter.otlp.proto.http import Compression
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+
+    otlp_traces_endpoint = os.getenv("OTLP_TRACES_ENDPOINT")
+    base_url = base_url or otlp_traces_endpoint
+    session = requests.Session()
+    return OTLPSpanExporter(
+        endpoint=base_url,
+        session=session,
+        compression=Compression.Gzip,
+    )
+
+
+def is_valid_attribute_value(k, v):
+    valid = True
+    if not v:
+        valid = False
+    valid = isinstance(v, (str, bool, int, float)) or \
+        (isinstance(v, Sequence) and
+            all(isinstance(i, (str, bool, int, float)) for i in v))
+    if not valid:
+        logger.debug(f"value of attribute[{k}] is invalid: {v}")
+    return valid

chapter9/gaia-experience/AWorld/aworld/trace/propagator/__init__.py

@@ -0,0 +1,76 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+import traceback
+from contextvars import ContextVar, Token
+from aworld.trace.base import TraceContext, Propagator
+from aworld.trace.propagator.w3c import W3CTraceContextPropagator
+from aworld.trace.baggage.sofa_tracer import SofaTracerBaggagePropagator
+from aworld.trace.baggage.w3c import W3CBaggagePropagator
+from aworld.logs.util import logger
+
+
+class CompositePropagator(Propagator):
+    """
+    Composite propagator.
+    """
+
+    def __init__(self, propagators: list[Propagator]):
+        self._propagators = propagators
+
+    def extract(self, carrier: dict) -> TraceContext:
+        trace_context = None
+        for propagator in self._propagators:
+            try:
+                context = propagator.extract(carrier)
+                if context and not trace_context:
+                    trace_context = context
+            except Exception:
+                stack_trace = traceback.format_exc()
+                logger.error(
+                    f"Failed to extract trace context: {stack_trace}, propagator: {propagator.__class__.__name__}")
+        return trace_context
+
+    def inject(self, trace_context: TraceContext, carrier: dict) -> None:
+        for propagator in self._propagators:
+            propagator.inject(trace_context, carrier)
+
+
+_GLOBAL_TRACE_PROPAGATOR = CompositePropagator(
+    [W3CTraceContextPropagator(), SofaTracerBaggagePropagator(), W3CBaggagePropagator()])
+
+
+def get_global_trace_propagator():
+    return _GLOBAL_TRACE_PROPAGATOR
+
+
+class TraceContextHolder:
+    def __init__(self):
+        self._var = ContextVar("current_trace_context", default=None)
+
+    def set(self, trace_context: TraceContext) -> Token:
+        if not trace_context or not trace_context.trace_id or not trace_context.span_id:
+            return None
+        token = self._var.set(trace_context)
+        return token
+
+    def get_and_clear(self) -> TraceContext:
+        try:
+            value = self._var.get()
+        except LookupError:
+            return self._var.get(None)
+        finally:
+            self._var.set(None)
+        return value
+
+    def get(self) -> TraceContext:
+        return self._var.get()
+
+    def reset(self, token: Token):
+        self._var.reset(token)
+
+
+_GLOBAL_TRACE_CONTEXT = TraceContextHolder()
+
+
+def get_global_trace_context():
+    return _GLOBAL_TRACE_CONTEXT

chapter9/gaia-experience/AWorld/aworld/trace/propagator/carrier.py

@@ -0,0 +1,42 @@
+from typing import TypeVar
+from aworld.trace.base import Carrier
+from aworld.logs.util import logger
+
+T = TypeVar("T")
+
+
+class ListTupleCarrier(Carrier):
+
+    def __init__(self, headers: list[tuple[str, T]]):
+        self.headers = headers
+
+    def get(self, key: str) -> T:
+        for header, value in self.headers:
+            header_str = header.decode(
+                'utf-8') if isinstance(header, bytes) else header
+            key_str = key.decode('utf-8') if isinstance(key, bytes) else key
+            if header_str.lower() == key_str.lower():
+                return value.decode('utf-8') if isinstance(value, bytes) else value
+        return None
+
+    def set(self, key: str, value: T) -> None:
+        for i, (header, _) in enumerate(self.headers):
+            header_str = header.decode(
+                'utf-8') if isinstance(header, bytes) else header
+            key_str = key.decode('utf-8') if isinstance(key, bytes) else key
+            if header_str.lower() == key_str.lower():
+                self.headers[i] = (header, value)
+                return
+        self.headers.append((key, value))
+
+
+class DictCarrier(Carrier):
+    def __init__(self, headers: dict[str, T]):
+        self.headers = headers
+
+    def get(self, key: str) -> T:
+        return self.headers.get(key)
+
+    def set(self, key: str, value: T) -> None:
+        logger.info(f"set header {key}={value}")
+        self.headers[key] = value

chapter9/gaia-experience/AWorld/aworld/trace/propagator/w3c.py

@@ -0,0 +1,135 @@
+import re
+from typing import Tuple, List
+from aworld.logs.util import logger
+from aworld.trace.base import Propagator, Carrier, TraceContext
+
+
+class W3CTraceContextPropagator(Propagator):
+    """
+    OtelPropagator is a Propagator that extracts and injects using w3c TraceContext's headers.
+        carrier = {
+            "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01",
+            "tracestate": "congo=t61rcWkgMzE",
+            "baggage": "key1=value1,key2=value2"
+        }
+    """
+    _STATE_KEY_FORMAT = (
+        r"[a-z][_0-9a-z\-\*\/]{0,255}|"
+        r"[a-z0-9][_0-9a-z\-\*\/]{0,240}@[a-z][_0-9a-z\-\*\/]{0,13}"
+    )
+    _STATE_VALUE_FORMAT = (
+        r"[\x20-\x2b\x2d-\x3c\x3e-\x7e]{0,255}[\x21-\x2b\x2d-\x3c\x3e-\x7e]"
+    )
+    _state_delimiter_pattern = re.compile(r"[ \t]*,[ \t]*")
+    _state_member_pattern = re.compile(
+        f"({_STATE_KEY_FORMAT})(=)({_STATE_VALUE_FORMAT})[ \t]*")
+
+    _TRACEPARENT_HEADER_NAME = "traceparent"
+    _TRACESTATE_HEADER_NAME = "tracestate"
+    _TRACEPARENT_HEADER_FORMAT = (
+        "^[ \t]*([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})"
+        + "(-.*)?[ \t]*$"
+    )
+    _TRACEPARENT_HEADER_FORMAT_RE = re.compile(_TRACEPARENT_HEADER_FORMAT)
+
+    def extract(self, carrier: Carrier) -> TraceContext:
+        """
+        Extract trace context from carrier.
+        Args:
+            carrier: The carrier to extract trace context from.
+        Returns:
+            A dict of trace context.
+        """
+        header = carrier.get(self._TRACEPARENT_HEADER_NAME) or carrier.get(
+            'HTTP_' + self._TRACEPARENT_HEADER_NAME.upper())
+
+        if header is None:
+            return None
+
+        match = re.search(self._TRACEPARENT_HEADER_FORMAT_RE, header)
+        if not match:
+            return None
+
+        version: str = match.group(1)
+        trace_id: str = match.group(2)
+        span_id: str = match.group(3)
+        trace_flags: str = match.group(4)
+
+        logger.debug(
+            f"extract trace_id: {trace_id}, span_id: {span_id}, trace_flags: {trace_flags}, version: {version}")
+
+        if trace_id == "0" * 32 or span_id == "0" * 16:
+            return None
+        if version == "00":
+            if match.group(5):  # type: ignore
+                return None
+        if version == "ff":
+            return None
+
+        state_header = carrier.get(self._TRACESTATE_HEADER_NAME) or carrier.get(
+            'HTTP_' + self._TRACESTATE_HEADER_NAME.upper())
+        return TraceContext(
+            trace_id=trace_id,
+            span_id=span_id,
+            trace_flags=trace_flags,
+            version=version,
+            attributes=(self._extract_state_from_header(state_header))
+        )
+
+    def inject(self, trace_context: TraceContext, carrier: Carrier) -> None:
+        """
+        Inject trace context into carrier.
+        Args:
+            context: The trace context to inject.
+            carrier: The carrier to inject trace context into.
+        """
+        attribute_copy = trace_context.attributes.copy()
+        version: str = trace_context.version
+        trace_flags: str = trace_context.trace_flags
+        trace_id = trace_context.trace_id
+        span_id = trace_context.span_id
+        logger.debug(
+            f"inject trace_id: {trace_id}, span_id: {span_id}, trace_flags: {trace_flags}, version: {version}")
+        if (not trace_id or trace_id == "0" * 32
+                or not span_id or span_id == "0" * 16):
+            return
+
+        if isinstance(trace_id, int):
+            trace_id = format(trace_id, "032x")
+        if isinstance(span_id, int):
+            span_id = format(span_id, "016x")
+        traceparent_string = f"{version}-{trace_id}-{span_id}-{trace_flags}"
+        carrier.set(self._TRACEPARENT_HEADER_NAME, traceparent_string)
+        tracestate_string = ",".join(
+            f"{key}={value}" for key, value in attribute_copy.items())
+        if tracestate_string:
+            carrier.set(self._TRACESTATE_HEADER_NAME, tracestate_string)
+
+    def _extract_state_from_header(self, header: str) -> dict:
+        """
+        Extract state from header.
+        Args:
+            header: The header to extract state from.
+        Returns:
+            A dict of state.
+        """
+        if header is None:
+            return {}
+        state = {}
+        members: List[str] = re.split(self._state_delimiter_pattern, header)
+        for member in members:
+            # empty members are valid, but no need to process further.
+            if not member:
+                continue
+            match = self._state_member_pattern.fullmatch(member)
+            if not match:
+                logger.warning(
+                    "Member doesn't match the w3c identifiers format {member}")
+                return state
+            groups: Tuple[str, ...] = match.groups()
+            key, _eq, value = groups
+            # duplicate keys are not legal in header
+            if key in state:
+                return state
+            state[key] = value
+        return state

chapter9/gaia-experience/AWorld/aworld/trace/rewrite_ast.py

@@ -0,0 +1,259 @@
+from __future__ import annotations
+
+import ast
+import uuid
+import time
+from pathlib import Path
+from collections import deque
+from functools import partial
+from typing import TYPE_CHECKING, Any, Callable, ContextManager, cast
+
+from aworld.trace.base import AttributeValueType
+from aworld.trace.constants import ATTRIBUTES_MESSAGE_TEMPLATE_KEY
+
+if TYPE_CHECKING:
+    from .context_manager import TraceManager
+    from .auto_trace import not_auto_trace
+
+
+def compile_source(
+        tree: ast.AST, filename: str, module_name: str, trace_manager: TraceManager, min_duration_ns: int
+) -> Callable[[dict[str, Any]], None]:
+    """Compile a modified AST of the module's source code in the module's namespace.
+
+    Returns a function which accepts module globals and executes the compiled code.
+
+    The modified AST wraps the body of every function definition in `with context_factories[index]():`.
+    `context_factories` is added to the module's namespace as `aworld_<uuid>`.
+    `index` is a different constant number for each function definition.
+    """
+
+    context_factories_var_name = f'aworld_{uuid.uuid4().hex}'
+    # The variable name for storing context_factors in the module's namespace.
+
+    context_factories: list[Callable[[], ContextManager[Any]]] = []
+    tree = rewrite_ast(tree, filename, context_factories_var_name, module_name, trace_manager, context_factories,
+                       min_duration_ns)
+    assert isinstance(tree, ast.Module)  # for type checking
+    # dont_inherit=True is necessary to prevent the module from inheriting the __future__ import from this module.
+    code = compile(tree, filename, 'exec', dont_inherit=True)
+
+    def execute(globs: dict[str, Any]):
+        globs[context_factories_var_name] = context_factories
+        exec(code, globs, globs)
+
+    return execute
+
+
+def rewrite_ast(
+        tree: ast.AST,
+        filename: str,
+        context_factories_var_name: str,
+        module_name: str,
+        trace_manager: TraceManager,
+        context_factories: list[Callable[[], ContextManager[Any]]],
+        min_duration_ns: int,
+) -> ast.AST:
+    transformer = AutoTraceTransformer(
+        context_factories_var_name, filename, module_name, trace_manager, context_factories, min_duration_ns
+    )
+    return transformer.visit(tree)
+
+
+class AutoTraceTransformer(ast.NodeTransformer):
+    """Trace all encountered functions except those explicitly marked with `@no_auto_trace`."""
+
+    def __init__(
+            self,
+            context_factories_var_name: str,
+            filename: str,
+            module_name: str,
+            trace_manager: TraceManager,
+            context_factories: list[Callable[[], ContextManager[Any]]],
+            min_duration_ns: int,
+    ):
+        self._context_factories_var_name = context_factories_var_name
+        self._filename = filename
+        self._module_name = module_name
+        self._trace_manager = trace_manager
+        self._context_factories = context_factories
+        self._min_duration_ns = min_duration_ns
+        self._qualname_stack: list[str] = []
+
+    def visit_ClassDef(self, node: ast.ClassDef):
+        """Visit a class definition and rewrite its methods."""
+
+        if self.check_not_auto_trace(node):
+            return node
+
+        self._qualname_stack.append(node.name)
+        node = cast(ast.ClassDef, self.generic_visit(node))
+        self._qualname_stack.pop()
+        return node
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST:
+        """Visit a function definition and rewrite it."""
+
+        if self.check_not_auto_trace(node):
+            return node
+
+        self._qualname_stack.append(node.name)
+        qualname = '.'.join(self._qualname_stack)
+        self._qualname_stack.append('<locals>')
+        self.generic_visit(node)
+        self._qualname_stack.pop()  # <locals>
+        self._qualname_stack.pop()  # node.name
+        return self.rewrite_function(node, qualname)
+
+    def check_not_auto_trace(self, node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) -> bool:
+        """Return true if the node has a `@not_auto_trace` decorator."""
+        return any(
+            (
+                    isinstance(node, ast.Name)
+                    and node.id == not_auto_trace.__name__
+                # or (
+                #     isinstance(node, ast.Attribute)
+                #     and node.attr == not_auto_trace.__name__
+                #     and isinstance(node.value, ast.Name)
+                #     and node.value.id == xxx.__name__
+                # )
+            )
+            for node in node.decorator_list
+        )
+
+    def rewrite_function(self, node: ast.FunctionDef | ast.AsyncFunctionDef, qualname: str) -> ast.AST:
+        """Rewrite a function definition to trace its execution."""
+
+        if has_yield(node):
+            return node
+
+        body = node.body.copy()
+        new_body: list[ast.stmt] = []
+        if (
+                body
+                and isinstance(body[0], ast.Expr)
+                and isinstance(body[0].value, ast.Constant)
+                and isinstance(body[0].value.value, str)
+        ):
+            new_body.append(body.pop(0))
+
+        if not body or (
+                len(body) == 1
+                and (
+                        isinstance(body[0], ast.Pass)
+                        or (isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant))
+                )
+        ):
+            return node
+
+        span = ast.With(
+            items=[
+                ast.withitem(
+                    context_expr=self.trace_context_method_call_node(node, qualname),
+                )
+            ],
+            body=body,
+            type_comment=node.type_comment,
+        )
+        new_body.append(span)
+
+        return ast.fix_missing_locations(
+            ast.copy_location(
+                type(node)(  # type: ignore
+                    name=node.name,
+                    args=node.args,
+                    body=new_body,
+                    decorator_list=node.decorator_list,
+                    returns=node.returns,
+                    type_comment=node.type_comment,
+                ),
+                node,
+            )
+        )
+
+    def trace_context_method_call_node(self, node: ast.FunctionDef | ast.AsyncFunctionDef, qualname: str) -> ast.Call:
+        """Return a method call to `context_factories[index]()`."""
+
+        index = len(self._context_factories)
+        span_factory = partial(
+            self._trace_manager._create_auto_span,  # type: ignore
+            *self.build_create_auto_span_args(qualname, node.lineno),
+        )
+        if self._min_duration_ns > 0:
+
+            timer = time.time_ns
+            min_duration = self._min_duration_ns
+
+            # This needs to be as fast as possible since it's the cost of auto-tracing a function
+            # that never actually gets instrumented because its calls are all faster than `min_duration`.
+            class MeasureTime:
+                __slots__ = 'start'
+
+                def __enter__(_self):
+                    _self.start = timer()
+
+                def __exit__(_self, *_):
+                    # the first call exceeding min_ruration will not be tracked, and subsequent calls will only be tracked
+                    if timer() - _self.start >= min_duration:
+                        self._context_factories[index] = span_factory
+
+            self._context_factories.append(MeasureTime)
+        else:
+            self._context_factories.append(span_factory)
+
+        # This node means:
+        #   context_factories[index]()
+        # where `context_factories` is a global variable with the name `self._context_factories_var_name`
+        # pointing to the `self.context_factories` list.
+        return ast.Call(
+            func=ast.Subscript(
+                value=ast.Name(id=self._context_factories_var_name, ctx=ast.Load()),
+                slice=ast.Index(value=ast.Constant(value=index)),  # type: ignore
+                ctx=ast.Load(),
+            ),
+            args=[],
+            keywords=[],
+        )
+
+    def build_create_auto_span_args(self, qualname: str, lineno: int) -> tuple[str, dict[str, AttributeValueType]]:
+        """Build the arguments for `create_auto_span`."""
+
+        stack_info = {
+            'code.filepath': get_filepath(self._filename),
+            'code.lineno': lineno,
+            'code.function': qualname,
+        }
+        attributes: dict[str, AttributeValueType] = {**stack_info}  # type: ignore
+
+        msg_template = f'Calling {self._module_name}.{qualname}'
+        attributes[ATTRIBUTES_MESSAGE_TEMPLATE_KEY] = msg_template
+
+        span_name = msg_template
+
+        return span_name, attributes
+
+
+def has_yield(node: ast.AST):
+    """Return true if the node has a yield statement."""
+
+    queue = deque([node])
+    while queue:
+        node = queue.popleft()
+        for child in ast.iter_child_nodes(node):
+            if isinstance(child, (ast.Yield, ast.YieldFrom)):
+                return True
+            if not isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
+                queue.append(child)
+
+
+def get_filepath(file: str):
+    """Return a dict with the filepath attribute."""
+
+    path = Path(file)
+    if path.is_absolute():
+        try:
+            path = path.relative_to(Path('.').resolve())
+        except ValueError:  # pragma: no cover
+            # happens if filename path is not within CWD
+            pass
+    return str(path)

chapter9/gaia-experience/AWorld/aworld/trace/server/__init__.py

@@ -0,0 +1,66 @@
+# coding: utf-8
+# Copyright (c) 2025 inclusionAI.
+import threading
+import os
+from fastapi.staticfiles import StaticFiles
+from .routes import setup_routes
+from aworld.logs.util import logger
+from aworld.utils.import_package import import_package
+
+GLOBAL_TRACE_SERVER = None
+
+
+class TraceServer:
+    def __init__(self, storage, port: int = 7079):
+        self._storage = storage
+        self._port = port
+        self._thread = None
+        self.app = None
+        self._started = False
+
+    def start(self):
+        self._thread = threading.Thread(target=self._start_app, daemon=True)
+        self._thread.start()
+        self._started = True
+
+    def join(self):
+        if self._thread:
+            self._thread.join()
+        else:
+            raise Exception("Trace server not started.")
+
+    def get_storage(self):
+        return self._storage
+
+    def is_started(self):
+        return self._started
+
+    def _start_app(self):
+        import_package('uvicorn')  # noqa
+        import uvicorn
+        app = setup_routes(self._storage)
+
+        webui_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../cmd/web/webui")
+        static_path = os.path.join(webui_path, "public")
+        app.mount("/static", StaticFiles(directory=static_path), name="static")
+
+        self.app = app
+        # app.run(port=self._port)
+        uvicorn.run(app, host="0.0.0.0", port=self._port, loop="asyncio")
+
+
+def set_trace_server(storage, port: int = 7079, start_server=False):
+    global GLOBAL_TRACE_SERVER
+    if GLOBAL_TRACE_SERVER is None:
+        GLOBAL_TRACE_SERVER = TraceServer(storage, port)
+    if GLOBAL_TRACE_SERVER.is_started():
+        setup_routes(storage)
+        return
+    if start_server:
+        GLOBAL_TRACE_SERVER.start()
+
+
+def get_trace_server():
+    if GLOBAL_TRACE_SERVER is None:
+        logger.warning("No trace server has been set.")
+    return GLOBAL_TRACE_SERVER

chapter9/gaia-experience/AWorld/aworld/trace/server/routes.py

@@ -0,0 +1,54 @@
+from aworld.trace.opentelemetry.memory_storage import TraceStorage
+from aworld.utils.import_package import import_package
+from aworld.trace.server.util import build_trace_tree
+
+import_package('fastapi')  # noqa
+from fastapi import FastAPI
+from fastapi.responses import JSONResponse, RedirectResponse
+
+app = FastAPI()
+current_storage = None
+routes_setup = False
+
+
+def setup_routes(storage: TraceStorage):
+    global current_storage
+    current_storage = storage
+
+    global routes_setup
+
+    if routes_setup:
+        return app
+
+    @app.get("/")
+    async def root():
+        return RedirectResponse("/static/trace_ui.html")
+
+    @app.get('/api/trace/list')
+    async def traces():
+        trace_data = []
+        for trace_id in current_storage.get_all_traces():
+            spans = current_storage.get_all_spans(trace_id)
+            spans_sorted = sorted(spans, key=lambda x: x.start_time)
+            trace_tree = build_trace_tree(spans_sorted)
+            trace_data.append({
+                'trace_id': trace_id,
+                'root_span': trace_tree,
+            })
+            response = {
+                "data": trace_data
+            }
+        return JSONResponse(content=response)
+
+    @app.get('/api/traces/{trace_id}')
+    async def get_trace(trace_id):
+        spans = current_storage.get_all_spans(trace_id)
+        spans_sorted = sorted(spans, key=lambda x: x.start_time)
+        trace_tree = build_trace_tree(spans_sorted)
+        return JSONResponse(content={
+            'trace_id': trace_id,
+            'root_span': trace_tree,
+        })
+
+    routes_setup = True
+    return app

chapter9/gaia-experience/AWorld/aworld/trace/server/util.py

@@ -0,0 +1,241 @@
+import uuid
+from aworld.logs.util import logger
+from aworld.trace.opentelemetry.memory_storage import SpanModel
+from aworld.trace.constants import RunType, SPAN_NAME_PREFIX_EVENT_AGENT
+from aworld.trace.instrumentation import semconv
+
+
+def build_trace_tree(spans: list[SpanModel]):
+    spans_dict = {span.span_id: span.dict() for span in spans}
+    for span in list(spans_dict.values()):
+        parent_id = span['parent_id'] if span['parent_id'] else None
+        if parent_id:
+            parent_span = spans_dict.get(parent_id)
+            if not parent_span:
+                logger.warning(f"span[{parent_id}] not be exported")
+                parent_span = {
+                    'span_id': parent_id,
+                    'trace_id': span['trace_id'],
+                    'name': 'Pengding-Span',
+                    'start_time': span['start_time'],
+                    'end_time': span['end_time'],
+                    'duration_ms': span['duration_ms'],
+                    'attributes': {},
+                    'status': {},
+                    'parent_id': None,
+                    'run_type': 'OTHER'
+                }
+                spans_dict[parent_id] = parent_span
+            if 'children' not in parent_span:
+                parent_span['children'] = []
+            parent_span['children'].append(span)
+
+    root_spans = [span for span in spans_dict.values()
+                  if span['parent_id'] is None]
+    return root_spans
+
+
+def _get_agent_show_name(span: dict):
+    agent_name_prefix = SPAN_NAME_PREFIX_EVENT_AGENT
+    name = span.get("name")
+    if name and name.startswith(agent_name_prefix):
+        name = name[len(agent_name_prefix):]
+    if name and '---' in name:
+        name = name.split('---', 1)[0]
+    return name
+
+
+def _remove_span_detail(root_spans: list):
+    keys_to_keep = {'span_id', 'show_name', 'task_group_id', 'event_id'}
+    for span in root_spans:
+        keys_to_remove = [key for key in span.keys() if key not in keys_to_keep]
+        for key in keys_to_remove:
+            span.pop(key, None)
+        if 'children' in span:
+            _remove_span_detail(span['children'])
+
+
+def _get_top_task_nodes(spans_dict):
+    task_nodes = [
+        span for span in spans_dict.values()
+        if span.get('name', '').startswith('task.')
+    ]
+    top_task_nodes = []
+    for task_node in task_nodes:
+        parent_id = task_node.get('parent_id')
+        is_top = True
+
+        while parent_id:
+            parent_span = spans_dict.get(parent_id)
+            if not parent_span:
+                break
+
+            if parent_span.get('name', '').startswith('task.'):
+                is_top = False
+                break
+
+            parent_id = parent_span.get('parent_id')
+
+        if is_top:
+            top_task_nodes.append(task_node)
+
+    return top_task_nodes
+
+
+def _get_root_nodes(edges):
+    sources = set()
+    targets = set()
+    for edge in edges:
+        sources.add(edge['source'])
+        targets.add(edge['target'])
+    return sources - targets
+
+
+def _build_graph(root_spans: list):
+    nodes = []
+    edges = []
+
+    group_id_counter = 0
+
+    def __process_group_span(parent_spans, group_id, group_spans):
+        nonlocal group_id_counter
+        group_id_counter += 1
+        # add group node
+        group_node = {
+            'span_id': f'group_{group_id_counter}',
+            'group_id': group_id,
+            'show_name': 'Task Group'
+        }
+        nodes.append(group_node)
+
+        # add edges from parent_spans to group node
+        for parent_span in parent_spans:
+            edges.append({
+                'source': parent_span['span_id'],
+                'target': group_node['span_id']
+            })
+
+        # add edges from group node to children spans
+        last_spans = []
+        for child in group_spans:
+            edges.append({
+                'source': group_node['span_id'],
+                'target': child['span_id']
+            })
+            last_spans.extend(__process_span(child))
+        return last_spans
+
+    def __process_span(span):
+        nonlocal group_id_counter
+        nodes.append(span)
+        if 'children' in span:
+            groups = {}
+            for child in span['children']:
+                group_id = child.get('task_group_id', id(child))
+                if group_id not in groups:
+                    groups[group_id] = []
+                groups[group_id].append(child)
+
+            last_spans = [span]  # The leaf nodes of the current subtree
+            for group_id, group_spans in groups.items():
+                if len(group_spans) > 1:
+                    parent_spans = last_spans
+                    last_spans = __process_group_span(parent_spans, group_id, group_spans)
+                else:
+                    child_span = group_spans[0]
+                    # add edges from last_spans to child
+                    for prev_node in last_spans:
+                        edges.append({
+                            'source': prev_node['span_id'],
+                            'target': child_span['span_id']
+                        })
+                    last_spans = __process_span(child_span)
+        return last_spans
+
+    for span in root_spans:
+        __process_span(span)
+
+    return {
+        'nodes': nodes,
+        'edges': edges
+    }
+
+
+def get_agent_flow(trace_id):
+    from aworld.trace.server import get_trace_server
+
+    storage = get_trace_server().get_storage()
+    spans = storage.get_all_spans(trace_id)
+    spans_dict = {span.span_id: span.dict() for span in spans}
+    children_spans = []
+    top_task_nodes = _get_top_task_nodes(spans_dict)
+
+    filtered_spans = {}
+    for span_id, span in spans_dict.items():
+        if span.get('is_event', False) and span.get('run_type') == RunType.AGNET.value:
+            span['show_name'] = _get_agent_show_name(span)
+            span['event_id'] = span.get('attributes', {}).get('event.id')
+            filtered_spans[span_id] = span
+
+    sub_task_spans = []
+    for span in list(filtered_spans.values()):
+        skip_this_span = False
+        parent_id = span['parent_id'] if span['parent_id'] else None
+
+        while parent_id and parent_id not in filtered_spans:
+            parent_span = spans_dict.get(parent_id)
+            if parent_span and parent_span.get('run_type') == RunType.TASK.value:
+                # if str(parent_span['attributes'].get(semconv.TASK_IS_SUB_TASK)).lower() == 'true':
+                #     sub_task_spans.append(span)
+                #     skip_this_span = True
+                #     break
+                # else:
+                # print(f"parent_span_name: {parent_span['name']}")
+                span['task_group_id'] = parent_span['attributes'].get(semconv.TASK_GROUP_ID)
+            parent_id = parent_span['parent_id'] if parent_span and parent_span['parent_id'] else None
+
+        if skip_this_span:
+            continue
+        if parent_id:
+            parent_span = filtered_spans.get(parent_id)
+            if not parent_span:
+                continue
+
+            if 'children' not in parent_span:
+                parent_span['children'] = []
+            parent_span['children'].append(span)
+            children_spans.append(span)
+
+    filtered_span_list = [span for span in filtered_spans.values() if span not in sub_task_spans]
+    root_spans = [span for span in filtered_span_list
+                  if span not in children_spans]
+
+    data = _build_graph(root_spans)
+    _remove_span_detail(data["nodes"])
+
+    # add query start node
+    _add_query_node(data, top_task_nodes)
+    return data
+
+def _add_query_node(data, top_task_nodes):
+    top_task_node = top_task_nodes[0] if top_task_nodes else None
+    if top_task_node:
+        start_node_span_id = f'{uuid.uuid4().hex[:16]}'
+        data['nodes'].append({
+            'span_id': start_node_span_id,
+            'show_name': top_task_node['attributes'].get(semconv.TASK_INPUT),
+        })
+        root_nodes = _get_root_nodes(data['edges'])
+        if root_nodes:
+            for root_node in root_nodes:
+                data['edges'].append({
+                    'source': start_node_span_id,
+                    'target': root_node,
+                })
+        else:
+            for root_node in data['nodes']:
+                if root_node['span_id'] != start_node_span_id:
+                    data['edges'].append({
+                        'source': start_node_span_id,
+                        'target': root_node['span_id'],
+                    })

chapter9/gaia-experience/AWorld/aworld/trace/span_cosumer.py

@@ -0,0 +1,43 @@
+from abc import ABC, abstractmethod
+from typing import Sequence
+from aworld.trace.base import Span
+
+
+class SpanConsumer(ABC):
+    """SpanConsumer is a protocol that represents a consumer for spans.
+    """
+    @abstractmethod
+    def consume(self, spans: Sequence[Span]) -> None:
+        """Consumes a span.
+        Args:
+            spans: The span to consume.
+        """
+
+
+_SPAN_CONSUMER_REGISTRY = {}
+
+
+def register_span_consumer(default_kwargs=None) -> None:
+    """Registers a span consumer.
+    Args:
+        default_kwargs: A dictionary of default keyword arguments to pass to the span consumer.
+    """
+
+    default_kwargs = default_kwargs or {}
+
+    def decorator(cls):
+        _SPAN_CONSUMER_REGISTRY[cls.__name__] = (cls, default_kwargs)
+        return cls
+
+    return decorator
+
+
+def get_span_consumers() -> Sequence[SpanConsumer]:
+    """Returns a list of span consumers.
+    Returns:
+        A list of span consumers.
+    """
+    return [
+        cls(**kwargs)
+        for cls, kwargs in _SPAN_CONSUMER_REGISTRY.values()
+    ]

chapter9/gaia-experience/AWorld/aworld/trace/stack_info.py

@@ -0,0 +1,91 @@
+import inspect
+import sys
+import aworld.trace as atrace
+from types import CodeType, FrameType
+from typing import Optional, TypedDict, Union
+from functools import lru_cache
+from pathlib import Path
+
+StackInfo = TypedDict('StackInfo', {'code.filepath': str, 'code.lineno': int, 'code.function': str}, total=False)
+
+NON_USER_CODE_PREFIXES: tuple[str, ...] = ()
+
+def add_non_user_code_prefix(path: Union[str, Path]) -> None:
+    global NON_USER_CODE_PREFIXES
+    path = str(Path(path).absolute())
+    NON_USER_CODE_PREFIXES += (path,)  
+
+add_non_user_code_prefix(Path(inspect.__file__).parent)
+add_non_user_code_prefix(Path(atrace.__file__).parent)
+
+def get_user_stack_info() -> StackInfo:
+    """Get the stack info for the first calling frame in user code.
+
+    See is_user_code for details.
+    Returns an empty dict if no such frame is found.
+    """
+    frame, _stacklevel = get_user_frame_and_stacklevel()
+    if frame:
+        return get_stack_info_from_frame(frame)
+    return {}
+
+
+def get_user_frame_and_stacklevel() -> tuple[Optional[FrameType], int]:
+    """Get the first calling frame in user code and a corresponding stacklevel that can be passed to `warnings.warn`.
+
+    See is_user_code for details.
+    Returns `(None, 0)` if no such frame is found.
+    """
+    frame = inspect.currentframe()
+    stacklevel = 0
+    while frame:
+        if is_user_code(frame.f_code):
+            return frame, stacklevel
+        frame = frame.f_back
+        stacklevel += 1
+    return None, 0
+
+def get_stack_info_from_frame(frame: FrameType) -> StackInfo:
+    return {
+        **get_code_object_info(frame.f_code),
+        'code.lineno': frame.f_lineno,
+    }
+
+@lru_cache(maxsize=2048)
+def get_code_object_info(code: CodeType) -> StackInfo:
+    result = get_filepath_attribute(code.co_filename)
+    if code.co_name != '<module>':  # pragma: no branch
+        result['code.function'] = code.co_qualname if sys.version_info >= (3, 11) else code.co_name
+    result['code.lineno'] = code.co_firstlineno
+    return result
+
+def get_filepath_attribute(file: str) -> StackInfo:
+    path = Path(file)
+    if path.is_absolute():
+        try:
+            path = path.relative_to(Path('.').resolve())
+        except ValueError:  # pragma: no cover
+            # happens if filename path is not within CWD
+            pass
+    return {'code.filepath': str(path)}
+
+@lru_cache(maxsize=8192)
+def is_user_code(code: CodeType) -> bool:
+    """Check if the code object is from user code.
+
+    A code object is not user code if:
+    - It is from a file in
+        - the standard library
+        - site-packages (specifically wherever opentelemetry is installed)
+        - an unknown location (e.g. a dynamically generated code object) indicated by a filename starting with '<'
+
+    - It is a list/dict/set comprehension.
+        These are artificial frames only created before Python 3.12,
+        and they are always called directly from the enclosing function so it makes sense to skip them.
+        On the other hand, generator expressions and lambdas might be called far away from where they are defined.
+    """
+    return not (
+        str(Path(code.co_filename).absolute()).startswith(NON_USER_CODE_PREFIXES)
+        or code.co_filename.startswith('<')
+        or code.co_name in ('<listcomp>', '<dictcomp>', '<setcomp>')
+    )