ai-agent-book 精选快照(<2MB 代码与文档,来自 github.com/bojieli/ai-agent-book)
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
# coding: utf-8
|
||||
# Copyright (c) 2025 inclusionAI.
|
||||
|
||||
from aworld.utils.import_package import import_package, import_packages
|
||||
@@ -0,0 +1,66 @@
|
||||
# coding: utf-8
|
||||
# Copyright (c) 2025 inclusionAI.
|
||||
|
||||
import asyncio
|
||||
from functools import wraps
|
||||
from typing import Callable, Optional, Union, Any, Dict
|
||||
|
||||
|
||||
class Functionable:
|
||||
def __init__(self, function: Callable[..., Any], *args: Any, **kwargs: Dict[str, Any]) -> None:
|
||||
self.function = function
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
self.done: bool = False
|
||||
self.error: bool = False
|
||||
self.result: Optional[Any] = None
|
||||
self.exception: Optional[Exception] = None
|
||||
|
||||
def __call__(self) -> None:
|
||||
try:
|
||||
self.result = self.function(*self.args, **self.kwargs)
|
||||
except Exception as e:
|
||||
self.error = True
|
||||
self.exception = e
|
||||
self.done = True
|
||||
|
||||
def call(self):
|
||||
self.__call__()
|
||||
|
||||
|
||||
def async_decorator(*func, delay: Optional[Union[int, float]] = 0.5) -> Callable:
|
||||
def wrapper(function: Callable[..., Any]) -> Callable[..., Any]:
|
||||
@wraps(function)
|
||||
async def inner_wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
sleep_time = 0 if delay is None else delay
|
||||
task = Functionable(function, *args, **kwargs)
|
||||
# TODO: Use thread pool to process task
|
||||
task.call()
|
||||
if task.error:
|
||||
raise task.exception
|
||||
await asyncio.sleep(sleep_time)
|
||||
return task.result
|
||||
|
||||
return inner_wrapper
|
||||
|
||||
if not func:
|
||||
return wrapper
|
||||
else:
|
||||
if asyncio.iscoroutinefunction(func[0]):
|
||||
# coroutine function, return itself
|
||||
return func[0]
|
||||
return wrapper(func[0])
|
||||
|
||||
def async_func(function: Callable[..., Any]) -> Callable[..., Any]:
|
||||
@wraps(function)
|
||||
async def inner_wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
task = Functionable(function, *args, **kwargs)
|
||||
task.call()
|
||||
if task.error:
|
||||
raise task.exception
|
||||
return task.result
|
||||
|
||||
if asyncio.iscoroutinefunction(function):
|
||||
# coroutine function, return itself
|
||||
return function
|
||||
return inner_wrapper
|
||||
@@ -0,0 +1,370 @@
|
||||
# coding: utf-8
|
||||
# Copyright (c) 2025 inclusionAI.
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import pkgutil
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
from types import FunctionType
|
||||
from typing import Callable, Any, Tuple, List, Iterator, Dict, Union
|
||||
|
||||
from aworld.logs.util import logger
|
||||
|
||||
|
||||
def convert_to_snake(name: str) -> str:
|
||||
"""Class name convert to snake."""
|
||||
if '_' not in name:
|
||||
name = re.sub(r'([a-z])([A-Z])', r'\1_\2', name)
|
||||
return name.lower()
|
||||
|
||||
|
||||
def snake_to_camel(snake):
|
||||
words = snake.split('_')
|
||||
return ''.join([w.capitalize() for w in words])
|
||||
|
||||
|
||||
def is_abstract_method(cls, method_name):
|
||||
method = getattr(cls, method_name)
|
||||
return (hasattr(method, '__isabstractmethod__') and method.__isabstractmethod__) or (
|
||||
isinstance(method, FunctionType) and hasattr(
|
||||
method, '__abstractmethods__') and method in method.__abstractmethods__)
|
||||
|
||||
|
||||
def override_in_subclass(name: str, sub_cls: object, base_cls: object) -> bool:
|
||||
"""Judge whether a subclass overrides a specified method.
|
||||
|
||||
Args:
|
||||
name: The method name of sub class and base class
|
||||
sub_cls: Specify subclasses of the base class.
|
||||
base_cls: The parent class of the subclass.
|
||||
|
||||
Returns:
|
||||
Overwrite as true in subclasses, vice versa.
|
||||
"""
|
||||
if not issubclass(sub_cls, base_cls):
|
||||
logger.warning(f"{sub_cls} is not sub class of {base_cls}")
|
||||
return False
|
||||
|
||||
if sub_cls == base_cls and hasattr(sub_cls, name) and not is_abstract_method(sub_cls, name):
|
||||
return True
|
||||
|
||||
this_method = getattr(sub_cls, name)
|
||||
base_method = getattr(base_cls, name)
|
||||
return this_method is not base_method
|
||||
|
||||
|
||||
def convert_to_subclass(obj, subclass):
|
||||
obj.__class__ = subclass
|
||||
return obj
|
||||
|
||||
|
||||
def _walk_to_root(path: str) -> Iterator[str]:
|
||||
"""Yield directories starting from the given directory up to the root."""
|
||||
if not os.path.exists(path):
|
||||
yield ''
|
||||
|
||||
if os.path.isfile(path):
|
||||
path = os.path.dirname(path)
|
||||
|
||||
last_dir = None
|
||||
current_dir = os.path.abspath(path)
|
||||
while last_dir != current_dir:
|
||||
yield current_dir
|
||||
parent_dir = os.path.abspath(os.path.join(current_dir, os.path.pardir))
|
||||
last_dir, current_dir = current_dir, parent_dir
|
||||
|
||||
|
||||
def find_file(filename: str) -> str:
|
||||
"""Find file from the folders for the given file.
|
||||
|
||||
NOTE: Current running path priority, followed by the execution file path, and finally the aworld package path.
|
||||
|
||||
Args:
|
||||
filename: The file name that you want to search.
|
||||
"""
|
||||
|
||||
def run_dir():
|
||||
try:
|
||||
main = __import__('__main__', None, None, fromlist=['__file__'])
|
||||
return os.path.dirname(main.__file__)
|
||||
except ModuleNotFoundError:
|
||||
return os.getcwd()
|
||||
|
||||
path = os.getcwd()
|
||||
if os.path.exists(os.path.join(path, filename)):
|
||||
path = os.getcwd()
|
||||
elif os.path.exists(os.path.join(run_dir(), filename)):
|
||||
path = run_dir()
|
||||
else:
|
||||
frame = inspect.currentframe()
|
||||
current_file = __file__
|
||||
|
||||
while frame.f_code.co_filename == current_file or not os.path.exists(
|
||||
frame.f_code.co_filename
|
||||
):
|
||||
assert frame.f_back is not None
|
||||
frame = frame.f_back
|
||||
frame_filename = frame.f_code.co_filename
|
||||
path = os.path.dirname(os.path.abspath(frame_filename))
|
||||
|
||||
for dirname in _walk_to_root(path):
|
||||
if not dirname:
|
||||
continue
|
||||
check_path = os.path.join(dirname, filename)
|
||||
if os.path.isfile(check_path):
|
||||
return check_path
|
||||
|
||||
return ''
|
||||
|
||||
|
||||
def search_in_module(module: object, base_classes: List[type]) -> List[Tuple[str, type]]:
|
||||
"""Find all classes that inherit from a specific base class in the module."""
|
||||
results = []
|
||||
for name, obj in inspect.getmembers(module, inspect.isclass):
|
||||
for base_class in base_classes:
|
||||
if issubclass(obj, base_class) and obj is not base_class:
|
||||
results.append((name, obj))
|
||||
return results
|
||||
|
||||
|
||||
def _scan_package(package_name: str, base_classes: List[type], results: List[Tuple[str, type]] = []):
|
||||
try:
|
||||
package = sys.modules[package_name]
|
||||
except:
|
||||
return
|
||||
|
||||
try:
|
||||
for sub_package, name, is_pkg in pkgutil.walk_packages(package.__path__):
|
||||
try:
|
||||
__import__(f"{package_name}.{name}")
|
||||
except:
|
||||
continue
|
||||
|
||||
if is_pkg:
|
||||
_scan_package(package_name + "." + name, base_classes, results)
|
||||
try:
|
||||
module = __import__(f"{package_name}.{name}", fromlist=[name])
|
||||
results.extend(search_in_module(module, base_classes))
|
||||
except:
|
||||
continue
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def scan_packages(package: str, base_classes: List[type]) -> List[Tuple[str, type]]:
|
||||
results = []
|
||||
_scan_package(package, base_classes, results)
|
||||
return results
|
||||
|
||||
|
||||
class ReturnThread(threading.Thread):
|
||||
def __init__(self, func, *args, **kwargs):
|
||||
threading.Thread.__init__(self)
|
||||
self.func = func
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
self.result = None
|
||||
self.daemon = True
|
||||
|
||||
def run(self):
|
||||
self.result = asyncio.run(self.func(*self.args, **self.kwargs))
|
||||
|
||||
|
||||
def asyncio_loop():
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
return loop
|
||||
|
||||
|
||||
def sync_exec(async_func: Callable[..., Any], *args, **kwargs):
|
||||
"""Async function to sync execution."""
|
||||
if not asyncio.iscoroutinefunction(async_func):
|
||||
return async_func(*args, **kwargs)
|
||||
|
||||
loop = asyncio_loop()
|
||||
if loop and loop.is_running():
|
||||
thread = ReturnThread(async_func, *args, **kwargs)
|
||||
thread.start()
|
||||
thread.join()
|
||||
result = thread.result
|
||||
else:
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
except Exception as e:
|
||||
logger.warning(f"get_event_loop fail. {e}")
|
||||
return asyncio.run(async_func(*args, **kwargs))
|
||||
result = loop.run_until_complete(async_func(*args, **kwargs))
|
||||
return result
|
||||
|
||||
|
||||
def nest_dict_counter(usage: Dict[str, Union[int, Dict[str, int]]],
|
||||
other: Dict[str, Union[int, Dict[str, int]]],
|
||||
ignore_zero: bool = True):
|
||||
"""Add counts from two dicts or nest dicts."""
|
||||
result = {}
|
||||
for elem, count in usage.items():
|
||||
# nest dict
|
||||
if isinstance(count, Dict):
|
||||
res = nest_dict_counter(usage[elem], other.get(elem, {}))
|
||||
result[elem] = res
|
||||
continue
|
||||
|
||||
newcount = count + other.get(elem, 0)
|
||||
if not ignore_zero or newcount > 0:
|
||||
result[elem] = newcount
|
||||
|
||||
for elem, count in other.items():
|
||||
if elem not in usage and not ignore_zero:
|
||||
result[elem] = count
|
||||
return result
|
||||
|
||||
|
||||
def get_class(module_class: str):
|
||||
import importlib
|
||||
|
||||
assert module_class
|
||||
module_class = module_class.strip()
|
||||
idx = module_class.rfind('.')
|
||||
if idx != -1:
|
||||
module = importlib.import_module(module_class[0:idx])
|
||||
class_names = module_class[idx + 1:].split(":")
|
||||
cls_obj = getattr(module, class_names[0])
|
||||
for inner_class_name in class_names[1:]:
|
||||
cls_obj = getattr(cls_obj, inner_class_name)
|
||||
return cls_obj
|
||||
else:
|
||||
raise Exception("{} can not find!".format(module_class))
|
||||
|
||||
|
||||
def new_instance(module_class: str, *args, **kwargs):
|
||||
"""Create module class instance based on module name."""
|
||||
return get_class(module_class)(*args, **kwargs)
|
||||
|
||||
|
||||
def load_module_by_path(module_name: str, file_path: str):
|
||||
"""Load python module from the file path."""
|
||||
file_path = str(Path(file_path).resolve())
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
except:
|
||||
logger.info(f"loading {module_name} fail from {file_path}, {traceback.format_exc()}")
|
||||
|
||||
|
||||
def retryable(tries: int = 3, delay: int = 1):
|
||||
def inner_retry(f):
|
||||
@wraps(f)
|
||||
def f_retry(*args, **kwargs):
|
||||
mtries, mdelay = tries, delay
|
||||
while mtries > 0:
|
||||
try:
|
||||
return f(*args, **kwargs)
|
||||
except Exception as e:
|
||||
msg = f"{str(e)}, Retrying in {mdelay} seconds..."
|
||||
logger.warning(msg)
|
||||
time.sleep(mdelay)
|
||||
mtries -= 1
|
||||
return f(*args, **kwargs)
|
||||
|
||||
return f_retry
|
||||
|
||||
return inner_retry
|
||||
|
||||
|
||||
def get_local_ip():
|
||||
try:
|
||||
# build UDP socket
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
# connect to an external address (no need to connect)
|
||||
s.connect(("8.8.8.8", 80))
|
||||
# get local IP
|
||||
local_ip = s.getsockname()[0]
|
||||
s.close()
|
||||
return local_ip
|
||||
except Exception:
|
||||
return "127.0.0.1"
|
||||
|
||||
|
||||
def replace_env_variables(config) -> Any:
|
||||
"""Replace environment variables in configuration.
|
||||
|
||||
Environment variables should be in the format ${ENV_VAR_NAME}.
|
||||
|
||||
Args:
|
||||
config: Configuration to process (dict, list, or other value)
|
||||
|
||||
Returns:
|
||||
Processed configuration with environment variables replaced
|
||||
"""
|
||||
if isinstance(config, dict):
|
||||
for key, value in config.items():
|
||||
config[key] = replace_env_variables(value)
|
||||
elif isinstance(config, list):
|
||||
for i, item in enumerate(config):
|
||||
config[i] = replace_env_variables(item)
|
||||
elif isinstance(config, str):
|
||||
pattern = r'\${([^}]+)}'
|
||||
matches = re.findall(pattern, config)
|
||||
for env_var_name in matches:
|
||||
env_var_value = os.getenv(env_var_name, f"${{{env_var_name}}}")
|
||||
config = config.replace(f'${{{env_var_name}}}', env_var_value)
|
||||
if env_var_value != f"${{{env_var_name}}}":
|
||||
logger.info(f"Replaced ${{{env_var_name}}} with {env_var_value}")
|
||||
return config
|
||||
|
||||
|
||||
def get_local_hostname():
|
||||
"""
|
||||
Get the local hostname.
|
||||
First try `socket.gethostname()`, if it fails or returns an invalid value,
|
||||
then try reverse DNS lookup using local IP.
|
||||
"""
|
||||
try:
|
||||
hostname = socket.gethostname()
|
||||
# Simple validation - if hostname contains '.', consider it a valid FQDN (Fully Qualified Domain Name)
|
||||
if hostname and '.' in hostname:
|
||||
return hostname
|
||||
|
||||
# If hostname is not qualified, try reverse lookup via IP
|
||||
local_ip = get_local_ip()
|
||||
if local_ip:
|
||||
try:
|
||||
# Get hostname from IP
|
||||
hostname, _, _ = socket.gethostbyaddr(local_ip)
|
||||
return hostname
|
||||
except (socket.herror, socket.gaierror):
|
||||
# Reverse lookup failed, return original hostname or IP
|
||||
pass
|
||||
|
||||
# If all methods fail, return original gethostname() result or IP
|
||||
return hostname if hostname else local_ip
|
||||
|
||||
except Exception:
|
||||
# Final fallback strategy
|
||||
return "localhost"
|
||||
|
||||
|
||||
def load_mcp_config():
|
||||
"""Load MCP server configurations from config file."""
|
||||
|
||||
path_cwd = os.getcwd()
|
||||
mcp_path = os.path.join(path_cwd, "mcp.json")
|
||||
try:
|
||||
with open(mcp_path, "r") as f:
|
||||
return json.load(f)
|
||||
except Exception as err:
|
||||
logger.error(f"Error loading MCP config[{mcp_path}] err is : {err}")
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
# coding: utf-8
|
||||
# Copyright (c) 2025 inclusionAI.
|
||||
import os.path
|
||||
import time
|
||||
import sys
|
||||
import importlib
|
||||
import subprocess
|
||||
from importlib import metadata
|
||||
from aworld.logs.util import logger
|
||||
|
||||
|
||||
class ModuleAlias:
|
||||
def __init__(self, module):
|
||||
self.module = module
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.module, name)
|
||||
|
||||
|
||||
def is_package_installed(package_name: str, version: str = "") -> bool:
|
||||
"""
|
||||
Check if package is already installed and matches version if specified.
|
||||
|
||||
Args:
|
||||
package_name: Name of the package to check
|
||||
version: Required version of the package
|
||||
|
||||
Returns:
|
||||
bool: True if package is installed (and version matches if specified), False otherwise
|
||||
"""
|
||||
try:
|
||||
dist = metadata.distribution(package_name)
|
||||
|
||||
if version and dist.version != version:
|
||||
logger.info(f"Package {package_name} is installed but version {dist.version} "
|
||||
f"does not match required version {version}")
|
||||
return False
|
||||
|
||||
logger.info(f"Package {package_name} is already installed (version: {dist.version})")
|
||||
return True
|
||||
|
||||
except metadata.PackageNotFoundError:
|
||||
logger.info(f"Package {package_name} is not installed")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking if {package_name} is installed: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
def import_packages(packages: list[str]) -> dict:
|
||||
"""
|
||||
Import and install multiple packages
|
||||
|
||||
Args:
|
||||
packages: List of packages to import
|
||||
|
||||
Returns:
|
||||
dict: Dictionary mapping package names to imported modules
|
||||
"""
|
||||
modules = {}
|
||||
for package in packages:
|
||||
package_ = import_package(package)
|
||||
if package_:
|
||||
modules[package] = package_
|
||||
return modules
|
||||
|
||||
|
||||
def import_package(
|
||||
package_name: str,
|
||||
alias: str = '',
|
||||
install_name: str = '',
|
||||
version: str = '',
|
||||
installer: str = 'pip',
|
||||
timeout: int = 300,
|
||||
retry_count: int = 3,
|
||||
retry_delay: int = 5
|
||||
) -> object:
|
||||
"""
|
||||
Import and install package if not available.
|
||||
|
||||
Args:
|
||||
package_name: Name of the package to import
|
||||
alias: Alias to use for the imported module
|
||||
install_name: Name of the package to install (if different from import name)
|
||||
version: Required version of the package
|
||||
installer: Package installer to use ('pip' or 'conda')
|
||||
timeout: Installation timeout in seconds
|
||||
retry_count: Number of installation retries if install fails
|
||||
retry_delay: Delay between retries in seconds
|
||||
|
||||
Returns:
|
||||
Imported module
|
||||
|
||||
Raises:
|
||||
ValueError: If input parameters are invalid
|
||||
ImportError: If package cannot be imported or installed
|
||||
TimeoutError: If installation exceeds timeout
|
||||
"""
|
||||
# Validate input parameters
|
||||
if not package_name:
|
||||
raise ValueError("Package name cannot be empty")
|
||||
|
||||
if installer not in ['pip', 'conda']:
|
||||
raise ValueError(f"Unsupported installer: {installer}")
|
||||
|
||||
# Use package_name as install_name if not provided
|
||||
real_install_name = install_name if install_name else package_name
|
||||
|
||||
# First, check if we need to install the package
|
||||
need_install = False
|
||||
|
||||
# Try to import the module first
|
||||
try:
|
||||
logger.debug(f"Attempting to import {package_name}")
|
||||
module = importlib.import_module(package_name)
|
||||
logger.debug(f"Successfully imported {package_name}")
|
||||
|
||||
# If we successfully imported the module, check version if specified
|
||||
if version:
|
||||
try:
|
||||
# For packages with different import and install names,
|
||||
# we need to check the install name for version info
|
||||
installed_version = metadata.version(real_install_name)
|
||||
if installed_version != version:
|
||||
logger.warning(
|
||||
f"Package {real_install_name} version mismatch. "
|
||||
f"Required: {version}, Installed: {installed_version}"
|
||||
)
|
||||
need_install = True
|
||||
except metadata.PackageNotFoundError:
|
||||
logger.warning(f"Could not determine version for {real_install_name}")
|
||||
|
||||
# If no need to reinstall for version mismatch, return the module
|
||||
if not need_install:
|
||||
return ModuleAlias(module) if alias else module
|
||||
|
||||
except ImportError as import_err:
|
||||
logger.info(f"Could not import {package_name}: {str(import_err)}")
|
||||
# Check if the package is installed
|
||||
if not is_package_installed(real_install_name, version):
|
||||
need_install = True
|
||||
else:
|
||||
# If package is installed but import failed, there might be an issue with dependencies
|
||||
# or the package itself. Still, let's try to reinstall it.
|
||||
logger.warning(f"Package {real_install_name} is installed but import of {package_name} failed. "
|
||||
f"Will attempt reinstallation.")
|
||||
need_install = True
|
||||
|
||||
# Install the package if needed
|
||||
if need_install:
|
||||
logger.info(f"Installation needed for {real_install_name}")
|
||||
|
||||
# Attempt installation with retries
|
||||
for attempt in range(retry_count):
|
||||
try:
|
||||
cmd = _get_install_command(installer, real_install_name, version)
|
||||
logger.info(f"Installing {real_install_name} with command: {' '.join(cmd)}")
|
||||
_execute_install_command(cmd, timeout)
|
||||
|
||||
# Break out of retry loop if installation succeeds
|
||||
break
|
||||
|
||||
except (ImportError, TimeoutError, subprocess.SubprocessError) as e:
|
||||
if attempt < retry_count - 1:
|
||||
logger.warning(
|
||||
f"Installation attempt {attempt + 1} failed: {str(e)}. Retrying in {retry_delay} seconds...")
|
||||
time.sleep(retry_delay)
|
||||
else:
|
||||
logger.error(f"All installation attempts failed for {real_install_name}")
|
||||
raise ImportError(f"Failed to install {real_install_name} after {retry_count} attempts: {str(e)}")
|
||||
|
||||
# Try importing after installation
|
||||
try:
|
||||
logger.debug(f"Attempting to import {package_name} after installation")
|
||||
module = importlib.import_module(package_name)
|
||||
logger.debug(f"Successfully imported {package_name}")
|
||||
return ModuleAlias(module) if alias else module
|
||||
except ImportError as e:
|
||||
error_msg = f"Failed to import {package_name} even after installation of {real_install_name}: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
|
||||
|
||||
def _get_install_command(installer: str, package_name: str, version: str = "") -> list:
|
||||
"""
|
||||
Generate installation command based on specified installer.
|
||||
|
||||
Args:
|
||||
installer: Package installer to use ('pip' or 'conda')
|
||||
package_name: Name of the package to install
|
||||
version: Required version of the package
|
||||
|
||||
Returns:
|
||||
list: Command as a list of strings
|
||||
|
||||
Raises:
|
||||
ValueError: If unsupported installer is specified
|
||||
"""
|
||||
if installer == 'pip':
|
||||
# Use sys.executable to ensure the right Python interpreter is used
|
||||
pytho3 = os.path.basename(sys.executable)
|
||||
cmd = [sys.executable, '-m', 'pip', 'install', '--upgrade']
|
||||
if version:
|
||||
cmd.append(f'{package_name}=={version}')
|
||||
else:
|
||||
cmd.append(package_name)
|
||||
elif installer == 'conda':
|
||||
cmd = ['conda', 'install', '-y', package_name]
|
||||
if version:
|
||||
cmd.extend([f'={version}'])
|
||||
else:
|
||||
raise ValueError(f"Unsupported installer: {installer}")
|
||||
|
||||
return cmd
|
||||
|
||||
|
||||
def _execute_install_command(cmd: list, timeout: int) -> None:
|
||||
"""
|
||||
Execute package installation command.
|
||||
|
||||
Args:
|
||||
cmd: Installation command as list of strings
|
||||
timeout: Installation timeout in seconds
|
||||
|
||||
Raises:
|
||||
TimeoutError: If installation exceeds timeout
|
||||
ImportError: If installation fails
|
||||
"""
|
||||
logger.info(f"Executing: {' '.join(cmd)}")
|
||||
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE
|
||||
)
|
||||
|
||||
try:
|
||||
stdout, stderr = process.communicate(timeout=timeout)
|
||||
|
||||
# Log installation output for debugging
|
||||
if stdout:
|
||||
logger.debug(f"Installation stdout: {stdout.decode()}")
|
||||
if stderr:
|
||||
logger.debug(f"Installation stderr: {stderr.decode()}")
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
error_msg = f"Package installation timed out after {timeout} seconds"
|
||||
logger.error(error_msg)
|
||||
raise TimeoutError(error_msg)
|
||||
|
||||
if process.returncode != 0:
|
||||
error_msg = f"Installation failed with code {process.returncode}: {stderr.decode()}"
|
||||
logger.error(error_msg)
|
||||
raise ImportError(error_msg)
|
||||
|
||||
logger.info("Installation completed successfully")
|
||||
@@ -0,0 +1,651 @@
|
||||
# coding: utf-8
|
||||
"""
|
||||
oss.py
|
||||
Utility class for OSS (Object Storage Service) operations.
|
||||
Provides simple methods for data operations: upload, read, delete, update.
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import tempfile
|
||||
from typing import Optional, Dict, List, Any, Tuple, Union, BinaryIO, TextIO, IO, AnyStr
|
||||
|
||||
from aworld.utils import import_package
|
||||
from aworld.logs.util import logger
|
||||
|
||||
|
||||
class OSSClient:
|
||||
"""
|
||||
A utility class for OSS (Object Storage Service) operations.
|
||||
Provides methods for data operations: upload, read, delete, update.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
access_key_id: Optional[str] = None,
|
||||
access_key_secret: Optional[str] = None,
|
||||
endpoint: Optional[str] = None,
|
||||
bucket_name: Optional[str] = None,
|
||||
enable_export: Optional[bool] = None):
|
||||
"""
|
||||
Initialize OSSClient with credentials.
|
||||
|
||||
Args:
|
||||
access_key_id: OSS access key ID. If None, will try to get from environment variable OSS_ACCESS_KEY_ID
|
||||
access_key_secret: OSS access key secret. If None, will try to get from environment variable OSS_ACCESS_KEY_SECRET
|
||||
endpoint: OSS endpoint. If None, will try to get from environment variable OSS_ENDPOINT
|
||||
bucket_name: OSS bucket name. If None, will try to get from environment variable OSS_BUCKET_NAME
|
||||
enable_export: Whether to enable OSS export. If None, will try to get from environment variable EXPORT_REPLAY_TRACE_TO_OSS
|
||||
"""
|
||||
self.access_key_id = access_key_id or os.getenv('OSS_ACCESS_KEY_ID')
|
||||
self.access_key_secret = access_key_secret or os.getenv('OSS_ACCESS_KEY_SECRET')
|
||||
self.endpoint = endpoint or os.getenv('OSS_ENDPOINT')
|
||||
self.bucket_name = bucket_name or os.getenv('OSS_BUCKET_NAME')
|
||||
self.enable_export = enable_export if enable_export is not None else os.getenv("EXPORT_REPLAY_TRACE_TO_OSS",
|
||||
"false").lower() == "true"
|
||||
self.bucket = None
|
||||
self._initialized = False
|
||||
|
||||
def initialize(self) -> bool:
|
||||
"""
|
||||
Initialize the OSS client with the provided or environment credentials.
|
||||
|
||||
Returns:
|
||||
bool: True if initialization is successful, False otherwise
|
||||
"""
|
||||
if self._initialized:
|
||||
return True
|
||||
|
||||
if not self.enable_export:
|
||||
logger.info("OSS export is disabled. Set EXPORT_REPLAY_TRACE_TO_OSS=true to enable.")
|
||||
return False
|
||||
|
||||
if not all([self.access_key_id, self.access_key_secret, self.endpoint, self.bucket_name]):
|
||||
logger.warn(
|
||||
"Missing required OSS credentials. Please provide all required parameters or set environment variables.")
|
||||
return False
|
||||
|
||||
try:
|
||||
import_package("oss2")
|
||||
import oss2
|
||||
auth = oss2.Auth(self.access_key_id, self.access_key_secret)
|
||||
self.bucket = oss2.Bucket(auth, self.endpoint, self.bucket_name)
|
||||
self._initialized = True
|
||||
return True
|
||||
except ImportError:
|
||||
logger.warn("Failed to import oss2 module. Please install it with 'pip install oss2'.")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to initialize OSS client. Error: {str(e)}")
|
||||
return False
|
||||
|
||||
# ---- Basic Data Operation Methods ----
|
||||
|
||||
def upload_data(self, data: Union[IO[AnyStr], str, bytes, dict], oss_key: str) -> Optional[dict]:
|
||||
"""
|
||||
Upload data to OSS. Supports various types of data:
|
||||
- In-memory file objects (IO[AnyStr])
|
||||
- Strings (str)
|
||||
- Bytes (bytes)
|
||||
- Dictionaries (dict), will be automatically converted to JSON
|
||||
- File paths (str)
|
||||
|
||||
Args:
|
||||
data: Data to upload, can be a file object or other supported types
|
||||
oss_key: The key (path) in OSS where the data will be stored
|
||||
|
||||
Returns:
|
||||
dict: {"oss_key": oss_key, "oss_url": url} if successful, None otherwise
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized or export is disabled")
|
||||
return None
|
||||
|
||||
try:
|
||||
# Handle file objects
|
||||
if hasattr(data, 'read'):
|
||||
content = data.read()
|
||||
if isinstance(content, str):
|
||||
content = content.encode('utf-8')
|
||||
self.bucket.put_object(oss_key, content)
|
||||
logger.info(f"Successfully uploaded memory file to OSS: {oss_key}")
|
||||
return {"oss_key": oss_key, "oss_url": self.get_object_url(oss_key)}
|
||||
|
||||
# Handle dictionaries
|
||||
if isinstance(data, dict):
|
||||
content = json.dumps(data, ensure_ascii=False).encode('utf-8')
|
||||
self.bucket.put_object(oss_key, content)
|
||||
return {"oss_key": oss_key, "oss_url": self.get_object_url(oss_key)}
|
||||
|
||||
# Handle strings
|
||||
if isinstance(data, str):
|
||||
# Check if it's a file path
|
||||
if os.path.isfile(data):
|
||||
self.bucket.put_object_from_file(oss_key, data)
|
||||
logger.info(f"Successfully uploaded file {data} to OSS: {oss_key}")
|
||||
return {"oss_key": oss_key, "oss_url": self.get_object_url(oss_key)}
|
||||
# Otherwise treat as string content
|
||||
content = data.encode('utf-8')
|
||||
self.bucket.put_object(oss_key, content)
|
||||
return {"oss_key": oss_key, "oss_url": self.get_object_url(oss_key)}
|
||||
|
||||
# Handle bytes
|
||||
self.bucket.put_object(oss_key, data)
|
||||
logger.info(f"Successfully uploaded data to OSS: {oss_key}")
|
||||
return {"oss_key": oss_key, "oss_url": self.get_object_url(oss_key)}
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to upload data to OSS: {str(e)}")
|
||||
return None
|
||||
|
||||
def read_data(self, oss_key: str, as_json: bool = False) -> Union[bytes, dict, str, None]:
|
||||
"""
|
||||
Read data from OSS.
|
||||
|
||||
Args:
|
||||
oss_key: The key (path) in OSS of the data to read
|
||||
as_json: If True, parse the data as JSON and return a dict
|
||||
|
||||
Returns:
|
||||
The data as bytes, dict (if as_json=True), or None if failed
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized or export is disabled")
|
||||
return None
|
||||
|
||||
try:
|
||||
# Read data
|
||||
result = self.bucket.get_object(oss_key)
|
||||
data = result.read()
|
||||
|
||||
# Convert to string or JSON if requested
|
||||
if as_json:
|
||||
return json.loads(data)
|
||||
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to read data from OSS: {str(e)}")
|
||||
return None
|
||||
|
||||
def read_text(self, oss_key: str) -> Optional[str]:
|
||||
"""
|
||||
Read text data from OSS.
|
||||
|
||||
Args:
|
||||
oss_key: The key (path) in OSS of the text to read
|
||||
|
||||
Returns:
|
||||
str: The text data, or None if failed
|
||||
"""
|
||||
data = self.read_data(oss_key)
|
||||
if data is not None:
|
||||
try:
|
||||
return data.decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to decode data as UTF-8: {str(e)}")
|
||||
return None
|
||||
|
||||
def delete_data(self, oss_key: str) -> bool:
|
||||
"""
|
||||
Delete data from OSS.
|
||||
|
||||
Args:
|
||||
oss_key: The key (path) in OSS of the data to delete
|
||||
|
||||
Returns:
|
||||
bool: True if successful, False otherwise
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized or export is disabled")
|
||||
return False
|
||||
|
||||
try:
|
||||
self.bucket.delete_object(oss_key)
|
||||
logger.info(f"Successfully deleted data from OSS: {oss_key}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to delete data from OSS: {str(e)}")
|
||||
return False
|
||||
|
||||
def update_data(self, oss_key: str, data: Union[IO[AnyStr], str, bytes, dict]) -> Optional[dict]:
|
||||
"""
|
||||
Update data in OSS (overwrite).
|
||||
|
||||
Args:
|
||||
oss_key: The key (path) in OSS where the data will be updated
|
||||
data: Data to update, can be a file object or other supported types
|
||||
|
||||
Returns:
|
||||
dict: {"oss_key": oss_key, "oss_url": url} if successful, None otherwise
|
||||
"""
|
||||
return self.upload_data(data, oss_key)
|
||||
|
||||
def update_json(self, oss_key: str, update_dict: dict) -> Optional[dict]:
|
||||
"""
|
||||
Update a JSON file in OSS by merging with the existing content.
|
||||
|
||||
Args:
|
||||
oss_key: The key (path) in OSS of the JSON file to update
|
||||
update_dict: The dictionary to merge into the existing JSON
|
||||
|
||||
Returns:
|
||||
dict: {"oss_key": oss_key, "oss_url": url} if successful, None otherwise
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized or export is disabled")
|
||||
return None
|
||||
|
||||
try:
|
||||
# Read existing data
|
||||
existing = self.read_data(oss_key, as_json=True)
|
||||
if not isinstance(existing, dict):
|
||||
existing = {}
|
||||
existing.update(update_dict)
|
||||
content = json.dumps(existing, ensure_ascii=False).encode('utf-8')
|
||||
self.bucket.put_object(oss_key, content)
|
||||
return {"oss_key": oss_key, "oss_url": self.get_object_url(oss_key)}
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to update JSON data in OSS: {str(e)}")
|
||||
return None
|
||||
|
||||
# ---- File Operation Methods ----
|
||||
|
||||
def upload_file(self, local_file: str, oss_key: Optional[str] = None) -> Optional[dict]:
|
||||
"""
|
||||
Upload a local file to OSS.
|
||||
|
||||
Args:
|
||||
local_file: Path to the local file
|
||||
oss_key: The key (path) in OSS where the file will be stored. If None, use the file name
|
||||
|
||||
Returns:
|
||||
dict: {"oss_key": oss_key, "oss_url": url} if successful, None otherwise
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized or export is disabled")
|
||||
return None
|
||||
|
||||
if not os.path.isfile(local_file):
|
||||
logger.warn(f"Local file {local_file} does not exist or is not a file")
|
||||
return None
|
||||
|
||||
if oss_key is None:
|
||||
oss_key = os.path.basename(local_file)
|
||||
|
||||
try:
|
||||
self.bucket.put_object_from_file(oss_key, local_file)
|
||||
logger.info(f"Successfully uploaded file {local_file} to OSS: {oss_key}")
|
||||
return {"oss_key": oss_key, "oss_url": self.get_object_url(oss_key)}
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to upload file to OSS: {str(e)}")
|
||||
return None
|
||||
|
||||
def download_file(self, oss_key: str, local_file: str) -> bool:
|
||||
"""
|
||||
Download a file from OSS to local.
|
||||
|
||||
Args:
|
||||
oss_key: The key (path) in OSS of the file to download
|
||||
local_file: Path where the downloaded file will be saved
|
||||
|
||||
Returns:
|
||||
bool: True if successful, False otherwise
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized or export is disabled")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Ensure the directory exists
|
||||
os.makedirs(os.path.dirname(os.path.abspath(local_file)), exist_ok=True)
|
||||
|
||||
# Download the file
|
||||
self.bucket.get_object_to_file(oss_key, local_file)
|
||||
logger.info(f"Successfully downloaded {oss_key} to {local_file}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to download {oss_key} from OSS: {str(e)}")
|
||||
return False
|
||||
|
||||
def list_objects(self, prefix: str = "", delimiter: str = "") -> List[Dict[str, Any]]:
|
||||
"""
|
||||
List objects in the OSS bucket with the given prefix.
|
||||
|
||||
Args:
|
||||
prefix: Prefix to filter objects
|
||||
delimiter: Delimiter for hierarchical listing
|
||||
|
||||
Returns:
|
||||
List of objects with their properties
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized or export is disabled")
|
||||
return []
|
||||
|
||||
try:
|
||||
result = []
|
||||
for obj in self.bucket.list_objects(prefix=prefix, delimiter=delimiter).object_list:
|
||||
result.append({
|
||||
'key': obj.key,
|
||||
'size': obj.size,
|
||||
'last_modified': obj.last_modified
|
||||
})
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to list objects with prefix {prefix}: {str(e)}")
|
||||
return []
|
||||
|
||||
# ---- Advanced Operation Methods ----
|
||||
|
||||
def exists(self, oss_key: str) -> bool:
|
||||
"""
|
||||
Check if an object exists in OSS.
|
||||
|
||||
Args:
|
||||
oss_key: The key (path) in OSS to check
|
||||
|
||||
Returns:
|
||||
bool: True if the object exists, False otherwise
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized or export is disabled")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Use head_object to check if the object exists
|
||||
self.bucket.head_object(oss_key)
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
def copy_object(self, source_key: str, target_key: str) -> bool:
|
||||
"""
|
||||
Copy an object within the same bucket.
|
||||
|
||||
Args:
|
||||
source_key: The source object key
|
||||
target_key: The target object key
|
||||
|
||||
Returns:
|
||||
bool: True if successful, False otherwise
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized or export is disabled")
|
||||
return False
|
||||
|
||||
try:
|
||||
self.bucket.copy_object(self.bucket_name, source_key, target_key)
|
||||
logger.info(f"Successfully copied {source_key} to {target_key}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to copy {source_key} to {target_key}: {str(e)}")
|
||||
return False
|
||||
|
||||
def get_object_url(self, oss_key: str, expires: int = 3600) -> Optional[str]:
|
||||
"""
|
||||
返回公开读bucket的直链URL
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized or export is disabled")
|
||||
return None
|
||||
|
||||
return f"https://{self.bucket_name}.{self.endpoint}/{oss_key}"
|
||||
|
||||
def upload_directory(self, local_dir: str, oss_prefix: str = "") -> Tuple[bool, List[str]]:
|
||||
"""
|
||||
Upload an entire directory to OSS.
|
||||
|
||||
Args:
|
||||
local_dir: Path to the local directory
|
||||
oss_prefix: Prefix to prepend to all uploaded files
|
||||
|
||||
Returns:
|
||||
Tuple of (success, list of uploaded files)
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized or export is disabled")
|
||||
return False, []
|
||||
|
||||
if not os.path.isdir(local_dir):
|
||||
logger.warn(f"Local directory {local_dir} does not exist or is not a directory")
|
||||
return False, []
|
||||
|
||||
uploaded_files = []
|
||||
errors = []
|
||||
|
||||
for root, _, files in os.walk(local_dir):
|
||||
for file in files:
|
||||
local_file = os.path.join(root, file)
|
||||
rel_path = os.path.relpath(local_file, local_dir)
|
||||
oss_key = os.path.join(oss_prefix, rel_path).replace("\\", "/")
|
||||
|
||||
result = self.upload_file(local_file, oss_key)
|
||||
if result:
|
||||
uploaded_files.append(result)
|
||||
else:
|
||||
errors.append(local_file)
|
||||
|
||||
if errors:
|
||||
logger.warn(f"Failed to upload {len(errors)} files")
|
||||
return False, uploaded_files
|
||||
return True, uploaded_files
|
||||
|
||||
|
||||
def get_oss_client(access_key_id: Optional[str] = None,
|
||||
access_key_secret: Optional[str] = None,
|
||||
endpoint: Optional[str] = None,
|
||||
bucket_name: Optional[str] = None,
|
||||
enable_export: Optional[bool] = None) -> OSSClient:
|
||||
"""
|
||||
Factory function to create and initialize an OSSClient.
|
||||
|
||||
Args:
|
||||
access_key_id: OSS access key ID
|
||||
access_key_secret: OSS access key secret
|
||||
endpoint: OSS endpoint
|
||||
bucket_name: OSS bucket name
|
||||
enable_export: Whether to enable OSS export
|
||||
|
||||
Returns:
|
||||
OSSClient: An initialized OSSClient instance
|
||||
"""
|
||||
client = OSSClient(
|
||||
access_key_id=access_key_id,
|
||||
access_key_secret=access_key_secret,
|
||||
endpoint=endpoint,
|
||||
bucket_name=bucket_name,
|
||||
enable_export=enable_export
|
||||
)
|
||||
client.initialize()
|
||||
return client
|
||||
|
||||
|
||||
def get_full_url(self, oss_key: str, temp_url: bool = False, expires: int = 3600) -> str:
|
||||
"""
|
||||
生成OSS对象的完整URL
|
||||
|
||||
Args:
|
||||
oss_key: OSS对象的键
|
||||
temp_url: 是否生成带签名的临时URL
|
||||
expires: 临时URL的过期时间(秒)
|
||||
|
||||
Returns:
|
||||
str: 对象的完整URL
|
||||
"""
|
||||
if not self.initialize():
|
||||
logger.warn("OSS client not initialized")
|
||||
return None
|
||||
|
||||
try:
|
||||
if temp_url:
|
||||
# 生成带签名的临时URL
|
||||
return self.bucket.sign_url('GET', oss_key, expires)
|
||||
else:
|
||||
# 生成永久URL (公开可访问的对象)
|
||||
return f"https://{self.bucket_name}.{self.endpoint}/{oss_key}"
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to generate URL for {oss_key}: {str(e)}")
|
||||
return None
|
||||
|
||||
# ---- Test Cases ----
|
||||
if __name__ == "__main__":
|
||||
os.environ["OSS_ACCESS_KEY_ID"] = ""
|
||||
os.environ["OSS_ACCESS_KEY_SECRET"] = ""
|
||||
os.environ["OSS_ENDPOINT"] = ""
|
||||
os.environ["OSS_BUCKET_NAME"] = ""
|
||||
|
||||
access_key_id = os.environ.get("OSS_ACCESS_KEY_ID")
|
||||
access_key_secret = os.environ.get("OSS_ACCESS_KEY_SECRET")
|
||||
endpoint = os.environ.get("OSS_ENDPOINT")
|
||||
bucket_name = os.environ.get("OSS_BUCKET_NAME")
|
||||
"""
|
||||
OSS tool class test cases
|
||||
Note: Before running the tests, you need to set the following environment variables,
|
||||
or provide the parameters directly in the test code:
|
||||
- OSS_ACCESS_KEY_ID
|
||||
- OSS_ACCESS_KEY_SECRET
|
||||
- OSS_ENDPOINT
|
||||
- OSS_BUCKET_NAME
|
||||
- EXPORT_REPLAY_TRACE_TO_OSS=true
|
||||
"""
|
||||
import io
|
||||
import time
|
||||
|
||||
# Test configuration
|
||||
TEST_PREFIX = f"test/oss_utils_123" # Use timestamp to avoid conflicts
|
||||
|
||||
# Initialize client
|
||||
# Method 1: Using environment variables
|
||||
# oss_client = get_oss_client(enable_export=True)
|
||||
|
||||
# Method 2: Provide parameters directly
|
||||
oss_client = get_oss_client(
|
||||
access_key_id=access_key_id, # Replace with your actual access key ID
|
||||
access_key_secret=access_key_secret, # Replace with your actual access key secret
|
||||
endpoint=endpoint, # Replace with your actual OSS endpoint
|
||||
bucket_name=bucket_name, # Replace with your actual bucket name
|
||||
enable_export=True
|
||||
)
|
||||
import datetime
|
||||
|
||||
current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
#Test1: Upload string data
|
||||
print("\nTest 1: Upload string data")
|
||||
text_key_new = f"{TEST_PREFIX}/text2.txt"
|
||||
result = oss_client.upload_data(f"malai This is a test text:{current_time}", text_key_new)
|
||||
print(f"Upload string data: {'Success: ' + result['oss_url'] if result else 'Failed'}")
|
||||
print(f"Upload string data: {result}")
|
||||
content = oss_client.read_data(text_key_new)
|
||||
print(f"Read text data: {content}")
|
||||
|
||||
text_key = f"{TEST_PREFIX}/text1.txt"
|
||||
|
||||
|
||||
# Test 2: Upload dictionary data (automatically converted to JSON)
|
||||
print("\nTest 2: Upload dictionary data")
|
||||
json_key = f"{TEST_PREFIX}/data.json"
|
||||
data = {
|
||||
"name": "Test data",
|
||||
"values": [1, 2, 3],
|
||||
"nested": {
|
||||
"key": "value"
|
||||
}
|
||||
}
|
||||
result = oss_client.upload_data(data, json_key)
|
||||
print(f"Upload dictionary data: {'Success: ' + result['oss_url'] if result else 'Failed'}")
|
||||
content = oss_client.read_data(text_key)
|
||||
print(f"Read text data: {content}")
|
||||
|
||||
# Test 3: Upload in-memory binary file object
|
||||
print("\nTest 3: Upload in-memory binary file object")
|
||||
binary_key = f"{TEST_PREFIX}/binary.dat"
|
||||
binary_data = io.BytesIO(b"\x00\x01\x02\x03\x04")
|
||||
result = oss_client.upload_data(binary_data, binary_key)
|
||||
print(f"Upload binary file object: {'Success: ' + result['oss_url'] if result else 'Failed'}")
|
||||
content = oss_client.read_data(text_key)
|
||||
print(f"Read text data: {content}")
|
||||
|
||||
# Test 4: Upload in-memory text file object
|
||||
print("\nTest 4: Upload in-memory text file object")
|
||||
text_file_key = f"{TEST_PREFIX}/text_file.txt"
|
||||
text_file = io.StringIO("This is the content of an in-memory text file")
|
||||
result = oss_client.upload_data(text_file, text_file_key)
|
||||
print(f"Upload text file object: {'Success: ' + result['oss_url'] if result else 'Failed'}")
|
||||
content = oss_client.read_data(text_key)
|
||||
print(f"Read text data: {content}")
|
||||
|
||||
# Test 5: Create and upload temporary file
|
||||
print("\nTest 5: Create and upload temporary file")
|
||||
with tempfile.NamedTemporaryFile(delete=False) as tmp:
|
||||
tmp.write(b"This is the content of a temporary file")
|
||||
tmp_path = tmp.name
|
||||
|
||||
file_key = f"{TEST_PREFIX}/temp_file.txt"
|
||||
result = oss_client.upload_file(tmp_path, file_key)
|
||||
print(f"Upload temporary file: {'Success: ' + result['oss_url'] if result else 'Failed'}")
|
||||
os.unlink(tmp_path) # Delete temporary file
|
||||
|
||||
# Test 6: Read text data
|
||||
print("\nTest 6: Read text data")
|
||||
content = oss_client.read_text(text_key)
|
||||
print(f"Read text data: {content}")
|
||||
|
||||
# Test 7: Read JSON data
|
||||
print("\nTest 7: Read JSON data")
|
||||
json_content = oss_client.read_data(json_key, as_json=True)
|
||||
print(f"Read JSON data: {json_content}")
|
||||
|
||||
# Test 8: Update JSON data (merge method)
|
||||
print("\nTest 8: Update JSON data")
|
||||
update_data = {"updated": True, "timestamp": time.time()}
|
||||
result = oss_client.update_json(json_key, update_data)
|
||||
print(f"Update JSON data: {'Success: ' + result['oss_url'] if result else 'Failed'}")
|
||||
|
||||
# View updated JSON data
|
||||
updated_json = oss_client.read_data(json_key, as_json=True)
|
||||
print(f"Updated JSON data: {updated_json}")
|
||||
|
||||
# Test 9: Overwrite existing data
|
||||
print("\nTest 9: Overwrite existing data")
|
||||
result = oss_client.upload_data("This is the overwritten text", text_key)
|
||||
print(f"Overwrite existing data: {'Success: ' + result['oss_url'] if result else 'Failed'}")
|
||||
|
||||
# View overwritten data
|
||||
new_content = oss_client.read_text(text_key)
|
||||
print(f"Overwritten text data: {new_content}")
|
||||
|
||||
# Test 10: List objects
|
||||
print("\nTest 10: List objects")
|
||||
objects = oss_client.list_objects(prefix=TEST_PREFIX)
|
||||
print(f"Found {len(objects)} objects:")
|
||||
for obj in objects:
|
||||
print(f" - {obj['key']} (Size: {obj['size']} bytes, Modified: {obj['last_modified']})")
|
||||
|
||||
# Test 11: Generate temporary URL
|
||||
print("\nTest 11: Generate temporary URL")
|
||||
url = oss_client.get_object_url(text_key, expires=300) # 5 minutes expiration
|
||||
print(f"Temporary URL: {url}")
|
||||
|
||||
# Test 12: Copy object
|
||||
print("\nTest 12: Copy object")
|
||||
copy_key = f"{TEST_PREFIX}/copy_of_text.txt"
|
||||
result = oss_client.copy_object(text_key, copy_key)
|
||||
print(f"Copy object: {'Success: ' + copy_key if result else 'Failed'}")
|
||||
|
||||
# Test 13: Check if object exists
|
||||
print("\nTest 13: Check if object exists")
|
||||
exists = oss_client.exists(text_key)
|
||||
print(f"Object {text_key} exists: {exists}")
|
||||
|
||||
non_existent_key = f"{TEST_PREFIX}/non_existent.txt"
|
||||
exists = oss_client.exists(non_existent_key)
|
||||
print(f"Object {non_existent_key} exists: {exists}")
|
||||
|
||||
# # Test 14: Delete objects
|
||||
# print("\nTest 14: Delete objects")
|
||||
# for obj in objects:
|
||||
# success = oss_client.delete_data(obj['key'])
|
||||
# print(f"Delete object {obj['key']}: {'Success' if success else 'Failed'}")
|
||||
#
|
||||
# # Cleanup: Delete copied object (may not be included in the previous list)
|
||||
# oss_client.delete_data(copy_key)
|
||||
#
|
||||
# print("\nTests completed!")
|
||||
@@ -0,0 +1,177 @@
|
||||
# coding: utf-8
|
||||
# Copyright (c) 2025 inclusionAI.
|
||||
import asyncio
|
||||
import uuid
|
||||
from typing import Any, List, Dict
|
||||
|
||||
from aworld.agents.llm_agent import Agent
|
||||
from aworld.config import RunConfig
|
||||
from aworld.core.common import ActionModel, Observation
|
||||
from aworld.core.context.base import Context
|
||||
from aworld.core.task import Task, TaskResponse
|
||||
from aworld.output.outputs import Outputs
|
||||
from aworld.runners.utils import choose_runners, execute_runner
|
||||
|
||||
|
||||
async def exec_tool(tool_name: str,
|
||||
action_name: str,
|
||||
params: dict,
|
||||
agent_name: str,
|
||||
context: Context,
|
||||
sub_task: bool = False,
|
||||
outputs: Outputs = None,
|
||||
task_group_id: str = None) -> TaskResponse:
|
||||
"""Utility method for executing a tool in a task-oriented manner.
|
||||
|
||||
Args:
|
||||
tool_name: Name of tool, required.
|
||||
action_name: Action name of tool, required.
|
||||
params: Tool params, required.
|
||||
agent_name: Agent name, required, can be empty.
|
||||
context: Context in the runtime, required.
|
||||
sub_task: Is it a subtask with the main task set to False.
|
||||
outputs: The same outputs instance, required in subtask.
|
||||
task_group_id: ID of group of task.
|
||||
"""
|
||||
actions = [ActionModel(tool_name=tool_name, action_name=action_name, params=params, agent_name=agent_name)]
|
||||
task = Task(input=actions,
|
||||
context=context,
|
||||
is_sub_task=sub_task,
|
||||
group_id=task_group_id,
|
||||
session_id=context.session_id)
|
||||
if outputs:
|
||||
task.outputs = outputs
|
||||
runners = await choose_runners([task], agent_oriented=False)
|
||||
res = await execute_runner(runners, RunConfig(reuse_process=True))
|
||||
resp: TaskResponse = res.get(task.id)
|
||||
return resp
|
||||
|
||||
|
||||
async def exec_agent(question: Any,
|
||||
agent: Agent,
|
||||
context: Context,
|
||||
sub_task: bool = False,
|
||||
outputs: Outputs = None,
|
||||
task_group_id: str = None) -> TaskResponse:
|
||||
"""Utility method for executing an agent in a task-oriented manner.
|
||||
|
||||
Args:
|
||||
question: Problems handled by agents.
|
||||
agent: Defined intelligent agents that solve specific problems.
|
||||
context: Context in the runtime.
|
||||
sub_task: Is it a subtask with the main task set to False.
|
||||
outputs: The same outputs instance.
|
||||
task_group_id: ID of group of task.
|
||||
"""
|
||||
task_id = uuid.uuid1().hex
|
||||
# sub_task_context = await context.build_sub_context(question, task_id, agents = {agent.id(): agent})
|
||||
# logger.info(f"{context.task_id} build sub_task: {task_id}, sub_task_context: {sub_task_context}")
|
||||
task = Task(id=task_id,
|
||||
input=question,
|
||||
agent=agent,
|
||||
context=context,
|
||||
is_sub_task=sub_task,
|
||||
group_id=task_group_id,
|
||||
session_id=context.session_id)
|
||||
if outputs:
|
||||
task.outputs = outputs
|
||||
runners = await choose_runners([task])
|
||||
res = await execute_runner(runners, RunConfig(reuse_process=True))
|
||||
resp: TaskResponse = res.get(task.id)
|
||||
return resp
|
||||
|
||||
|
||||
async def exec_agents(questions: List[Any],
|
||||
agents: List[Agent],
|
||||
context: Context,
|
||||
sub_task: bool = False,
|
||||
task_group_id: str = None) -> List[ActionModel]:
|
||||
"""Execute the agent list with the questions, using asyncio.
|
||||
|
||||
Args:
|
||||
questions: Problems handled by agents.
|
||||
agents: Defined intelligent agents that solve specific problem.
|
||||
context: Context in the runtime.
|
||||
sub_task: Is it a subtask with the main task set to False.
|
||||
task_group_id: ID of group of task.
|
||||
"""
|
||||
tasks = []
|
||||
if agents:
|
||||
for idx, agent in enumerate(agents):
|
||||
tasks.append(asyncio.create_task(
|
||||
exec_agent(questions[idx], agent, context, sub_task=sub_task, task_group_id=task_group_id)))
|
||||
|
||||
results = await asyncio.gather(*tasks)
|
||||
res = []
|
||||
for idx, result in enumerate(results):
|
||||
if result.success:
|
||||
con = result.answer
|
||||
else:
|
||||
con = result.msg
|
||||
res.append(ActionModel(agent_name=agents[idx].id(), policy_info=con))
|
||||
return res
|
||||
|
||||
|
||||
async def exec_process_agents(question: Any,
|
||||
agents: List[Agent],
|
||||
context: Context,
|
||||
sub_task: bool = False,
|
||||
task_group_id: str = None):
|
||||
"""Execute the agent list with the same question, using new process.
|
||||
|
||||
NOTE: Mixing coroutines and processes may lead to unknown issues.
|
||||
|
||||
Args:
|
||||
question: Problems handled by agents.
|
||||
agents: Defined intelligent agents that solve specific problem.
|
||||
context: Context in the runtime.
|
||||
sub_task: Is it a subtask with the main task set to False.
|
||||
task_group_id: ID of group of task.
|
||||
"""
|
||||
tasks = []
|
||||
agent_map = {}
|
||||
if agents:
|
||||
for agent in agents:
|
||||
task = Task(input=question, agent=agent, context=context, is_sub_task=sub_task, group_id=task_group_id)
|
||||
agent_map[task.id] = agent.id()
|
||||
tasks.append(task)
|
||||
|
||||
if not tasks:
|
||||
raise RuntimeError("no task need to run.")
|
||||
|
||||
runners = await choose_runners(tasks)
|
||||
results = await execute_runner(runners, RunConfig(reuse_process=True))
|
||||
|
||||
res = []
|
||||
for key, result in results.items():
|
||||
res.append(ActionModel(agent_name=agent_map[key], policy_info=result))
|
||||
return res
|
||||
|
||||
|
||||
async def exec_tasks(tasks: List[Task], run_conf: RunConfig = RunConfig()) -> Dict[str, TaskResponse]:
|
||||
final_tasks = []
|
||||
# task list sequence-dependent execution
|
||||
if run_conf and run_conf.sequence_dependent:
|
||||
return await serial_exec_tasks(tasks=tasks, run_conf=run_conf)
|
||||
|
||||
for task in tasks:
|
||||
if not task.group_id:
|
||||
task.group_id = uuid.uuid4().hex
|
||||
final_tasks.append(task)
|
||||
runners = await choose_runners(final_tasks)
|
||||
return await execute_runner(runners, run_conf)
|
||||
|
||||
|
||||
async def serial_exec_tasks(tasks: List[Task], run_conf: RunConfig = RunConfig()) -> Dict[str, TaskResponse]:
|
||||
res = {}
|
||||
task_input = tasks[0].input
|
||||
for task in tasks:
|
||||
task.input = task_input
|
||||
runners = await choose_runners([task])
|
||||
res = await execute_runner(runners, run_conf)
|
||||
result: TaskResponse = res.get(task.id)
|
||||
if result.success:
|
||||
task_input = result.answer
|
||||
else:
|
||||
task_input = result.msg
|
||||
return res
|
||||
@@ -0,0 +1,44 @@
|
||||
# coding: utf-8
|
||||
# Copyright (c) 2025 inclusionAI.
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
class NumpyEncoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
if isinstance(obj, np.ndarray):
|
||||
return obj.tolist()
|
||||
return super().default(obj)
|
||||
|
||||
|
||||
def to_serializable(obj, _memo=None):
|
||||
if _memo is None:
|
||||
_memo = set()
|
||||
obj_id = id(obj)
|
||||
if obj_id in _memo:
|
||||
return str(obj)
|
||||
_memo.add(obj_id)
|
||||
|
||||
if isinstance(obj, dict):
|
||||
return {k: to_serializable(v, _memo) for k, v in obj.items()}
|
||||
elif isinstance(obj, (list, set)):
|
||||
return [to_serializable(i, _memo) for i in obj]
|
||||
elif hasattr(obj, "to_dict"):
|
||||
return obj.to_dict()
|
||||
elif hasattr(obj, "model_dump"):
|
||||
return obj.model_dump()
|
||||
elif hasattr(obj, "dict"):
|
||||
return obj.dict()
|
||||
elif hasattr(obj, "__dataclass_fields__"):
|
||||
return {field.name: to_serializable(getattr(obj, field.name), _memo)
|
||||
for field in obj.__dataclass_fields__.values()}
|
||||
elif hasattr(obj, "__dict__"):
|
||||
return {k: to_serializable(v, _memo) for k, v in obj.__dict__.items()
|
||||
if not k.startswith('_') and not callable(v)}
|
||||
else:
|
||||
try:
|
||||
json.dumps(obj)
|
||||
return obj
|
||||
except TypeError as e:
|
||||
raise RuntimeError(f"{e}")
|
||||
Reference in New Issue
Block a user