liqiang
b119135836
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
389 lines
19 KiB
Python
389 lines
19 KiB
Python
"""Real data, webhook, and browser execution tools for Experiment 4-3."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import signal
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
from config import Config
|
|
|
|
|
|
def _sha(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def _safe_output(path: str) -> Path:
|
|
candidate = Path(path)
|
|
if not candidate.is_absolute():
|
|
candidate = Path(Config.WORKSPACE_DIR) / candidate
|
|
candidate = candidate.resolve()
|
|
candidate.relative_to(Path(Config.WORKSPACE_DIR).resolve())
|
|
candidate.parent.mkdir(parents=True, exist_ok=True)
|
|
return candidate
|
|
|
|
|
|
class ExtendedTools:
|
|
async def excel_create_with_formula_and_screenshot(
|
|
self, output_path: str, rows: list[dict[str, Any]]
|
|
) -> dict[str, Any]:
|
|
"""Create a real XLSX, apply formulas, and render a screenshot via LibreOffice."""
|
|
from openpyxl import Workbook
|
|
|
|
target = _safe_output(output_path)
|
|
workbook = Workbook()
|
|
sheet = workbook.active
|
|
sheet.title = "Invoice"
|
|
sheet.append(["Item", "Quantity", "Unit price", "Total"])
|
|
for index, row in enumerate(rows, 2):
|
|
sheet.append([row["item"], float(row["quantity"]), float(row["unit_price"]),
|
|
f"=B{index}*C{index}"])
|
|
total_row = len(rows) + 2
|
|
sheet.cell(total_row, 3, "Grand total")
|
|
sheet.cell(total_row, 4, f"=SUM(D2:D{total_row - 1})")
|
|
sheet.freeze_panes = "A2"
|
|
sheet.column_dimensions["A"].width = 28
|
|
for column in ("B", "C", "D"):
|
|
sheet.column_dimensions[column].width = 16
|
|
workbook.save(target)
|
|
|
|
soffice = shutil.which("soffice") or shutil.which("libreoffice")
|
|
if not soffice:
|
|
return {"success": False, "error": "LibreOffice is required for formula rendering"}
|
|
started = time.perf_counter()
|
|
process = subprocess.run(
|
|
[soffice, "--headless", "--convert-to", "pdf", "--outdir",
|
|
str(target.parent), str(target)],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=120,
|
|
)
|
|
pdf = target.with_suffix(".pdf")
|
|
if process.returncode != 0 or not pdf.is_file():
|
|
return {"success": False, "error": process.stderr or process.stdout,
|
|
"returncode": process.returncode}
|
|
import fitz
|
|
|
|
document = fitz.open(pdf)
|
|
screenshot = target.with_suffix(".png")
|
|
document[0].get_pixmap(matrix=fitz.Matrix(1.5, 1.5), alpha=False).save(screenshot)
|
|
document.close()
|
|
return {
|
|
"success": True,
|
|
"xlsx": {"path": str(target), "bytes": target.stat().st_size, "sha256": _sha(target)},
|
|
"pdf": {"path": str(pdf), "bytes": pdf.stat().st_size, "sha256": _sha(pdf)},
|
|
"screenshot": {"path": str(screenshot), "bytes": screenshot.stat().st_size,
|
|
"sha256": _sha(screenshot)},
|
|
"formula_cells": [f"D{index}" for index in range(2, total_row + 1)],
|
|
"rows": len(rows),
|
|
"renderer": "LibreOffice headless + PyMuPDF",
|
|
"latency_seconds": round(time.perf_counter() - started, 3),
|
|
}
|
|
|
|
async def webhook_post(self, url: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
"""POST JSON to a real HTTPS webhook and retain response evidence."""
|
|
if not url.startswith("https://"):
|
|
return {"success": False, "error": "Only HTTPS webhook URLs are allowed"}
|
|
started = time.perf_counter()
|
|
async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
|
|
response = await client.post(url, json=payload)
|
|
try:
|
|
body = response.json()
|
|
except ValueError:
|
|
body = {"text": response.text[:2000]}
|
|
return {
|
|
"success": response.is_success,
|
|
"status": response.status_code,
|
|
"url": str(response.url),
|
|
"response": body,
|
|
"response_sha256": hashlib.sha256(response.content).hexdigest(),
|
|
"response_bytes": len(response.content),
|
|
"latency_seconds": round(time.perf_counter() - started, 3),
|
|
}
|
|
|
|
async def browser_navigate(self, url: str, screenshot_path: str) -> dict[str, Any]:
|
|
"""Navigate with real headless Chromium, extract content, and retain pixels."""
|
|
if not url.startswith("https://"):
|
|
return {"success": False, "error": "Only HTTPS URLs are allowed"}
|
|
target = _safe_output(screenshot_path)
|
|
started = time.perf_counter()
|
|
from playwright.async_api import async_playwright
|
|
|
|
async with async_playwright() as playwright:
|
|
browser = await playwright.chromium.launch(headless=True)
|
|
page = await browser.new_page(viewport={"width": 1280, "height": 720})
|
|
response = await page.goto(url, wait_until="networkidle", timeout=60000)
|
|
title = await page.title()
|
|
text = (await page.locator("body").inner_text())[:4000]
|
|
await page.screenshot(path=str(target), full_page=True)
|
|
await browser.close()
|
|
return {
|
|
"success": bool(response and response.ok and target.is_file()),
|
|
"url": url,
|
|
"status": response.status if response else None,
|
|
"title": title,
|
|
"body_text": text,
|
|
"screenshot": {"path": str(target), "bytes": target.stat().st_size,
|
|
"sha256": _sha(target)},
|
|
"browser": "Chromium via Playwright",
|
|
"latency_seconds": round(time.perf_counter() - started, 3),
|
|
}
|
|
|
|
async def virtual_desktop_execute(
|
|
self, url: str, screenshot_path: str, expected_title: str | None = None
|
|
) -> dict[str, Any]:
|
|
"""Drive headful Chromium through X11 keyboard events and retain pixels."""
|
|
if not url.startswith("https://"):
|
|
return {"success": False, "error": "Only HTTPS URLs are allowed"}
|
|
target = _safe_output(screenshot_path)
|
|
required = {
|
|
name: shutil.which(name)
|
|
for name in ("Xvfb", "xdotool", "ffmpeg")
|
|
}
|
|
chromium = shutil.which("chromium") or shutil.which("chromium-browser")
|
|
missing = [name for name, path in required.items() if not path]
|
|
if not chromium:
|
|
missing.append("chromium")
|
|
if missing:
|
|
return {"success": False, "error": f"Missing desktop executables: {missing}"}
|
|
|
|
display_number = next((
|
|
number for number in range(90, 130)
|
|
if not Path(f"/tmp/.X11-unix/X{number}").exists()
|
|
and not Path(f"/tmp/.X{number}-lock").exists()
|
|
), None)
|
|
if display_number is None:
|
|
return {"success": False, "error": "No free bounded X11 display number"}
|
|
display = f":{display_number}"
|
|
started = time.perf_counter()
|
|
xvfb_process: subprocess.Popen[bytes] | None = None
|
|
chromium_process: subprocess.Popen[bytes] | None = None
|
|
|
|
def stop(process: subprocess.Popen[bytes] | None) -> None:
|
|
if process is None or process.poll() is not None:
|
|
return
|
|
try:
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
process.wait(timeout=5)
|
|
except (ProcessLookupError, subprocess.TimeoutExpired):
|
|
try:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
|
|
try:
|
|
xvfb_process = subprocess.Popen(
|
|
[required["Xvfb"], display, "-screen", "0", "1280x720x24", "-nolisten", "tcp"],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
start_new_session=True,
|
|
)
|
|
socket_path = Path(f"/tmp/.X11-unix/X{display_number}")
|
|
for _ in range(50):
|
|
if socket_path.exists():
|
|
break
|
|
if xvfb_process.poll() is not None:
|
|
return {"success": False, "error": "Xvfb exited before creating its socket"}
|
|
time.sleep(0.1)
|
|
else:
|
|
return {"success": False, "error": "Xvfb did not become ready"}
|
|
|
|
env = {**os.environ, "DISPLAY": display}
|
|
with tempfile.TemporaryDirectory(prefix="exp4-computer-use-") as profile:
|
|
chromium_process = subprocess.Popen(
|
|
[chromium, "--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage",
|
|
f"--user-data-dir={profile}", "about:blank"],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
env=env, start_new_session=True,
|
|
)
|
|
window_id = ""
|
|
for _ in range(100):
|
|
search = subprocess.run(
|
|
[required["xdotool"], "search", "--onlyvisible", "--class", "chromium"],
|
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, env=env,
|
|
)
|
|
if search.stdout.strip():
|
|
window_id = search.stdout.splitlines()[0].strip()
|
|
break
|
|
if chromium_process.poll() is not None:
|
|
return {"success": False, "error": "Chromium exited before opening a window"}
|
|
time.sleep(0.1)
|
|
if not window_id:
|
|
return {"success": False, "error": "No visible Chromium window appeared"}
|
|
|
|
subprocess.run(
|
|
[required["xdotool"], "windowfocus", "--sync", window_id],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env,
|
|
)
|
|
input_receipts = []
|
|
for command in (
|
|
[required["xdotool"], "key", "--window", window_id, "ctrl+l"],
|
|
[required["xdotool"], "type", "--window", window_id, "--delay", "15", url],
|
|
[required["xdotool"], "key", "--window", window_id, "Return"],
|
|
):
|
|
completed = subprocess.run(
|
|
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env,
|
|
)
|
|
input_receipts.append({"operation": command[1], "returncode": completed.returncode})
|
|
if completed.returncode != 0:
|
|
return {"success": False, "error": completed.stderr.strip(),
|
|
"input_receipts": input_receipts}
|
|
|
|
title = ""
|
|
for _ in range(100):
|
|
title_result = subprocess.run(
|
|
[required["xdotool"], "getwindowname", window_id],
|
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, env=env,
|
|
)
|
|
title = title_result.stdout.strip()
|
|
if title and (not expected_title or expected_title in title):
|
|
break
|
|
time.sleep(0.1)
|
|
title_matched = bool(title and (not expected_title or expected_title in title))
|
|
capture = subprocess.run(
|
|
[required["ffmpeg"], "-nostdin", "-loglevel", "error", "-f", "x11grab",
|
|
"-video_size", "1280x720", "-i", display, "-frames:v", "1", "-y", str(target)],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env, timeout=30,
|
|
)
|
|
png_valid = target.is_file() and target.read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
|
|
return {
|
|
"success": capture.returncode == 0 and png_valid and title_matched,
|
|
"backend": "Xvfb + headful Chromium + xdotool",
|
|
"versions": {
|
|
"chromium": subprocess.run(
|
|
[chromium, "--version"], stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT, text=True,
|
|
).stdout.strip(),
|
|
"xdotool": subprocess.run(
|
|
[required["xdotool"], "-v"], stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT, text=True,
|
|
).stdout.strip(),
|
|
"ffmpeg": subprocess.run(
|
|
[required["ffmpeg"], "-version"], stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT, text=True,
|
|
).stdout.splitlines()[0],
|
|
},
|
|
"display": display,
|
|
"window_id": window_id,
|
|
"url_entered_via_os_keyboard": url,
|
|
"window_title": title,
|
|
"expected_title": expected_title,
|
|
"expected_title_matched": title_matched,
|
|
"input_receipts": input_receipts,
|
|
"screenshot": ({"path": str(target), "bytes": target.stat().st_size,
|
|
"sha256": _sha(target)} if png_valid else None),
|
|
"capture_returncode": capture.returncode,
|
|
"capture_error": capture.stderr.strip() or None,
|
|
"latency_seconds": round(time.perf_counter() - started, 3),
|
|
}
|
|
finally:
|
|
stop(chromium_process)
|
|
stop(xvfb_process)
|
|
|
|
async def virtual_mobile_execute(
|
|
self, container_name: str, screenshot_path: str
|
|
) -> dict[str, Any]:
|
|
"""Operate a real AndroidWorld emulator through ADB inside its container."""
|
|
if not re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", container_name):
|
|
return {"success": False, "error": "Invalid Docker container name"}
|
|
if not shutil.which("docker"):
|
|
return {"success": False, "error": "Docker is required"}
|
|
target = _safe_output(screenshot_path)
|
|
started = time.perf_counter()
|
|
|
|
def adb(*arguments: str, binary: bool = False) -> subprocess.CompletedProcess[Any]:
|
|
return subprocess.run(
|
|
["docker", "exec", container_name, "adb", *arguments],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
text=not binary, timeout=30,
|
|
)
|
|
|
|
inspect = subprocess.run(
|
|
["docker", "inspect", "--format", "{{.State.Running}} {{.Image}}", container_name],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
|
)
|
|
inspect_parts = inspect.stdout.strip().split(maxsplit=1)
|
|
if inspect.returncode != 0 or not inspect_parts or inspect_parts[0] != "true":
|
|
return {"success": False, "error": "AndroidWorld container is not running",
|
|
"container": container_name}
|
|
boot = adb("shell", "getprop", "sys.boot_completed")
|
|
devices = adb("devices", "-l")
|
|
if boot.stdout.strip() != "1" or " device " not in f" {devices.stdout} ":
|
|
return {"success": False, "error": "Android emulator is not boot-complete",
|
|
"container": container_name, "devices": devices.stdout.strip()}
|
|
|
|
model = adb("shell", "getprop", "ro.product.model").stdout.strip()
|
|
sdk = adb("shell", "getprop", "ro.build.version.sdk").stdout.strip()
|
|
focus_before = adb("shell", "dumpsys", "window").stdout
|
|
launch = adb("shell", "am", "start", "-W", "-a", "android.settings.WIFI_SETTINGS")
|
|
focus_settings = adb("shell", "dumpsys", "window").stdout
|
|
screenshot = adb("exec-out", "screencap", "-p", binary=True)
|
|
if screenshot.returncode == 0:
|
|
target.write_bytes(screenshot.stdout)
|
|
home = adb("shell", "input", "keyevent", "KEYCODE_HOME")
|
|
focus_home = adb("shell", "dumpsys", "window").stdout
|
|
png_valid = target.is_file() and target.read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
|
|
settings_focused = "com.android.settings" in focus_settings
|
|
launcher_focused = "launcher" in focus_home.lower()
|
|
return {
|
|
"success": all((launch.returncode == 0, settings_focused, home.returncode == 0,
|
|
launcher_focused, png_valid)),
|
|
"backend": "AndroidWorld Docker emulator + ADB",
|
|
"container": container_name,
|
|
"container_image_id": inspect_parts[1] if len(inspect_parts) > 1 else None,
|
|
"devices": devices.stdout.strip().splitlines(),
|
|
"boot_completed": boot.stdout.strip(),
|
|
"model": model,
|
|
"api_level": sdk,
|
|
"focus_before": next((line.strip() for line in focus_before.splitlines()
|
|
if "mCurrentFocus=" in line), None),
|
|
"settings_launch_returncode": launch.returncode,
|
|
"settings_activity": next((line.strip() for line in launch.stdout.splitlines()
|
|
if line.strip().startswith("Activity:")), None),
|
|
"settings_focus": next((line.strip() for line in focus_settings.splitlines()
|
|
if "mCurrentFocus=" in line), None),
|
|
"home_input_returncode": home.returncode,
|
|
"home_focus": next((line.strip() for line in focus_home.splitlines()
|
|
if "mCurrentFocus=" in line), None),
|
|
"screenshot": ({"path": str(target), "bytes": target.stat().st_size,
|
|
"sha256": _sha(target)} if png_valid else None),
|
|
"latency_seconds": round(time.perf_counter() - started, 3),
|
|
}
|
|
|
|
async def environment_capabilities(self) -> dict[str, Any]:
|
|
"""Report, without simulation, whether desktop/mobile backends are actually usable."""
|
|
docker_image = subprocess.run(
|
|
["docker", "image", "inspect",
|
|
"ghcr.io/anthropics/anthropic-quickstarts:computer-use-demo-latest"],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
).returncode == 0 if shutil.which("docker") else False
|
|
android_container = os.getenv("ANDROID_WORLD_CONTAINER", "")
|
|
active_devices: list[str] = []
|
|
if android_container and re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", android_container):
|
|
devices = subprocess.run(
|
|
["docker", "exec", android_container, "adb", "devices", "-l"],
|
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,
|
|
) if shutil.which("docker") else None
|
|
if devices and devices.returncode == 0:
|
|
active_devices = [line for line in devices.stdout.splitlines()[1:]
|
|
if " device " in f" {line} "]
|
|
return {
|
|
"success": True,
|
|
"computer_use_container_image_present": docker_image,
|
|
"computer_use_host_stack_present": all(shutil.which(name) for name in
|
|
("Xvfb", "xdotool", "ffmpeg"))
|
|
and bool(shutil.which("chromium") or
|
|
shutil.which("chromium-browser")),
|
|
"computer_use_active_session": False,
|
|
"android_world_container": android_container or None,
|
|
"android_world_adb_present": bool(active_devices),
|
|
"android_active_devices": active_devices,
|
|
"note": "Availability probe only; execution gates are established by the dedicated desktop and mobile action receipts.",
|
|
}
|