ai-agent-book 精选快照(<2MB 代码与文档,来自 github.com/bojieli/ai-agent-book)
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run and retain the bounded Experiment 6-7 native Computer Use trajectory.
|
||||
|
||||
This harness calls the pinned Anthropic Computer Use Demo's ``sampling_loop``.
|
||||
It is intended to run inside that Demo's locally built container with a host
|
||||
evidence directory mounted at ``/evidence``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
import traceback
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from computer_use_demo.loop import APIProvider, sampling_loop
|
||||
from computer_use_demo.tools import ToolResult
|
||||
|
||||
|
||||
TASK = (
|
||||
"Open Google, search for San Francisco weather today, and report the "
|
||||
"temperature and conditions. Do not sign in or change any external data."
|
||||
)
|
||||
MODEL = "claude-sonnet-4-5-20250929"
|
||||
TOOL_VERSION = "computer_use_20250124"
|
||||
ACTION_LIMIT = 25
|
||||
OUT = Path(os.environ.get("EXP96_EVIDENCE_DIR", "/evidence"))
|
||||
|
||||
|
||||
class ActionLimitReached(RuntimeError):
|
||||
"""Raised before an action beyond the experiment ceiling executes."""
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def sha256_bytes(value: bytes) -> str:
|
||||
return hashlib.sha256(value).hexdigest()
|
||||
|
||||
|
||||
def json_safe(value: Any) -> Any:
|
||||
if value is None or isinstance(value, (bool, int, float, str)):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return {str(k): json_safe(v) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [json_safe(v) for v in value]
|
||||
if hasattr(value, "model_dump"):
|
||||
return json_safe(value.model_dump())
|
||||
return repr(value)
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
if not os.environ.get("ANTHROPIC_API_KEY"):
|
||||
raise RuntimeError("ANTHROPIC_API_KEY is not set")
|
||||
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
screenshots = OUT / "screenshots"
|
||||
receipts = OUT / "api_receipts"
|
||||
screenshots.mkdir(exist_ok=True)
|
||||
receipts.mkdir(exist_ok=True)
|
||||
|
||||
started_at = utc_now()
|
||||
api_calls: list[dict[str, Any]] = []
|
||||
actions: list[dict[str, Any]] = []
|
||||
action_by_id: dict[str, dict[str, Any]] = {}
|
||||
refused_action: dict[str, Any] | None = None
|
||||
messages: list[dict[str, Any]] = [
|
||||
{"role": "user", "content": [{"type": "text", "text": TASK}]}
|
||||
]
|
||||
termination = "unknown"
|
||||
exception: dict[str, Any] | None = None
|
||||
|
||||
def api_response_callback(request: Any, response: Any, error: Any) -> None:
|
||||
index = len(api_calls) + 1
|
||||
request_body = b""
|
||||
try:
|
||||
request_body = request.content or b""
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
response_json = None
|
||||
response_status = getattr(response, "status_code", None)
|
||||
try:
|
||||
response_json = response.json()
|
||||
except Exception:
|
||||
if isinstance(response, (dict, list)):
|
||||
response_json = response
|
||||
|
||||
receipt_name = f"response-{index:02d}.json"
|
||||
if response_json is not None:
|
||||
(receipts / receipt_name).write_text(
|
||||
json.dumps(json_safe(response_json), indent=2, ensure_ascii=False)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
else:
|
||||
receipt_name = None
|
||||
|
||||
headers = getattr(response, "headers", {}) or {}
|
||||
api_calls.append(
|
||||
{
|
||||
"index": index,
|
||||
"observed_at": utc_now(),
|
||||
"request": {
|
||||
"method": getattr(request, "method", None),
|
||||
"url": str(getattr(request, "url", "")),
|
||||
"body_bytes": len(request_body),
|
||||
"body_sha256": sha256_bytes(request_body),
|
||||
"credential_header_present": bool(
|
||||
getattr(request, "headers", {}).get("x-api-key")
|
||||
),
|
||||
},
|
||||
"response": {
|
||||
"http_status": response_status,
|
||||
"request_id": headers.get("request-id")
|
||||
or headers.get("x-request-id"),
|
||||
"message_id": (
|
||||
response_json.get("id")
|
||||
if isinstance(response_json, dict)
|
||||
else None
|
||||
),
|
||||
"model": (
|
||||
response_json.get("model")
|
||||
if isinstance(response_json, dict)
|
||||
else None
|
||||
),
|
||||
"stop_reason": (
|
||||
response_json.get("stop_reason")
|
||||
if isinstance(response_json, dict)
|
||||
else None
|
||||
),
|
||||
"usage": (
|
||||
response_json.get("usage")
|
||||
if isinstance(response_json, dict)
|
||||
else None
|
||||
),
|
||||
"receipt": (
|
||||
f"api_receipts/{receipt_name}" if receipt_name else None
|
||||
),
|
||||
},
|
||||
"error_type": type(error).__name__ if error else None,
|
||||
"error": str(error) if error else None,
|
||||
}
|
||||
)
|
||||
|
||||
def output_callback(block: Any) -> None:
|
||||
nonlocal refused_action
|
||||
value = json_safe(block)
|
||||
if not isinstance(value, dict) or value.get("type") != "tool_use":
|
||||
return
|
||||
if len(actions) >= ACTION_LIMIT:
|
||||
refused_action = {
|
||||
"tool_use_id": value.get("id"),
|
||||
"tool": value.get("name"),
|
||||
"input": value.get("input"),
|
||||
"executed": False,
|
||||
"reason": "action_limit",
|
||||
}
|
||||
raise ActionLimitReached(
|
||||
f"refused action {ACTION_LIMIT + 1}; limit is {ACTION_LIMIT}"
|
||||
)
|
||||
record = {
|
||||
"index": len(actions) + 1,
|
||||
"tool_use_id": value.get("id"),
|
||||
"tool": value.get("name"),
|
||||
"input": value.get("input"),
|
||||
"executed": True,
|
||||
"result": None,
|
||||
}
|
||||
actions.append(record)
|
||||
action_by_id[str(value.get("id"))] = record
|
||||
|
||||
def tool_output_callback(result: ToolResult, tool_use_id: str) -> None:
|
||||
record = action_by_id[tool_use_id]
|
||||
image_path = None
|
||||
image_sha256 = None
|
||||
image_bytes = 0
|
||||
if result.base64_image:
|
||||
import base64
|
||||
|
||||
raw = base64.b64decode(result.base64_image)
|
||||
image_path = f"screenshots/action-{record['index']:02d}.png"
|
||||
(OUT / image_path).write_bytes(raw)
|
||||
image_sha256 = sha256_bytes(raw)
|
||||
image_bytes = len(raw)
|
||||
record["result"] = {
|
||||
"output": result.output,
|
||||
"error": result.error,
|
||||
"system": result.system,
|
||||
"screenshot": image_path,
|
||||
"screenshot_sha256": image_sha256,
|
||||
"screenshot_bytes": image_bytes,
|
||||
}
|
||||
|
||||
try:
|
||||
await sampling_loop(
|
||||
model=MODEL,
|
||||
provider=APIProvider.ANTHROPIC,
|
||||
system_prompt_suffix=(
|
||||
"This is a bounded, read-only evaluation. Do not sign in, accept "
|
||||
"agreements, submit forms, or modify external data. Use the GUI "
|
||||
"to perform the requested Google search and ground the final answer "
|
||||
"in the visible result. If Google presents a CAPTCHA or other human "
|
||||
"verification challenge, do not interact with it and do not ask the "
|
||||
"user to solve it. Instead, navigate directly to this reputable, "
|
||||
"read-only Open-Meteo current-weather endpoint: "
|
||||
"https://api.open-meteo.com/v1/forecast?latitude=37.7749&longitude="
|
||||
"-122.4194¤t=temperature_2m,weather_code&temperature_unit="
|
||||
"fahrenheit&timezone=America%2FLos_Angeles . Read the visible JSON, "
|
||||
"interpret its WMO weather code, identify Open-Meteo as the alternate "
|
||||
"source, and finish immediately. You have at most 25 actions total: "
|
||||
"once a credible current temperature and condition are visible, do "
|
||||
"not scroll or explore further; return the final answer."
|
||||
),
|
||||
messages=messages,
|
||||
output_callback=output_callback,
|
||||
tool_output_callback=tool_output_callback,
|
||||
api_response_callback=api_response_callback,
|
||||
api_key=os.environ["ANTHROPIC_API_KEY"],
|
||||
only_n_most_recent_images=3,
|
||||
max_tokens=4096,
|
||||
tool_version=TOOL_VERSION,
|
||||
thinking_budget=None,
|
||||
token_efficient_tools_beta=False,
|
||||
)
|
||||
termination = "model_finished"
|
||||
except ActionLimitReached as exc:
|
||||
termination = "action_limit"
|
||||
exception = {"type": type(exc).__name__, "message": str(exc)}
|
||||
except Exception as exc:
|
||||
termination = "error"
|
||||
exception = {
|
||||
"type": type(exc).__name__,
|
||||
"message": str(exc),
|
||||
"traceback": traceback.format_exc(),
|
||||
}
|
||||
|
||||
final_texts: list[str] = []
|
||||
for message in reversed(messages):
|
||||
if message.get("role") != "assistant":
|
||||
continue
|
||||
for block in message.get("content", []):
|
||||
value = json_safe(block)
|
||||
if isinstance(value, dict) and value.get("type") == "text":
|
||||
final_texts.append(value.get("text", ""))
|
||||
if final_texts:
|
||||
break
|
||||
|
||||
usage_totals: dict[str, int] = {}
|
||||
for call in api_calls:
|
||||
usage = call["response"].get("usage") or {}
|
||||
for key, value in usage.items():
|
||||
if isinstance(value, int):
|
||||
usage_totals[key] = usage_totals.get(key, 0) + value
|
||||
|
||||
final_stop_reason = (
|
||||
api_calls[-1]["response"].get("stop_reason") if api_calls else None
|
||||
)
|
||||
record = {
|
||||
"schema_version": 1,
|
||||
"experiment": "6-7",
|
||||
"status": "completed" if termination == "model_finished" else termination,
|
||||
"started_at": started_at,
|
||||
"finished_at": utc_now(),
|
||||
"task": TASK,
|
||||
"safety": {
|
||||
"read_only": True,
|
||||
"sign_in_allowed": False,
|
||||
"external_mutation_allowed": False,
|
||||
},
|
||||
"provider": "Anthropic API",
|
||||
"requested_model": MODEL,
|
||||
"observed_models": sorted(
|
||||
{
|
||||
call["response"]["model"]
|
||||
for call in api_calls
|
||||
if call["response"].get("model")
|
||||
}
|
||||
),
|
||||
"tool_version": TOOL_VERSION,
|
||||
"action_limit": ACTION_LIMIT,
|
||||
"actions_executed": len(actions),
|
||||
"termination": termination,
|
||||
"provider_stop_reason": final_stop_reason,
|
||||
"final_answer": "\n".join(reversed(final_texts)).strip(),
|
||||
"exception": exception,
|
||||
"refused_action": refused_action,
|
||||
"usage_totals": usage_totals,
|
||||
"api_calls": api_calls,
|
||||
"actions": actions,
|
||||
"runtime": {
|
||||
"python": sys.version,
|
||||
"platform": platform.platform(),
|
||||
"machine": platform.machine(),
|
||||
"source_commit": os.environ.get("EXP96_SOURCE_COMMIT"),
|
||||
"dockerfile_sha256": os.environ.get("EXP96_DOCKERFILE_SHA256"),
|
||||
"image_id": os.environ.get("EXP96_IMAGE_ID"),
|
||||
"base_image_digest": os.environ.get("EXP96_BASE_IMAGE_DIGEST"),
|
||||
},
|
||||
}
|
||||
(OUT / "trajectory.json").write_text(
|
||||
json.dumps(record, indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": record["status"],
|
||||
"api_calls": len(api_calls),
|
||||
"actions_executed": len(actions),
|
||||
"final_stop_reason": final_stop_reason,
|
||||
"final_answer": record["final_answer"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0 if termination == "model_finished" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
Reference in New Issue
Block a user