ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter10/multi-role-transfer/judge_comparison.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Blind, position-swapped quality review for retained Experiment 10-1 pairs."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from openai import OpenAI
+
+
+JUDGE_PROMPT = """你是独立的质量评审员。只根据用户任务和两个匿名候选答案评估：
+事实/计算正确性、是否满足全部约束、可审计性、清晰度。不要猜测隐藏思维，也不要因为
+候选声称调用工具就相信它；答案中没有证据就是缺失。严格只输出三行：
+WINNER: A 或 B 或 TIE
+SCORE_A: 0 到 4 的整数
+SCORE_B: 0 到 4 的整数
+
+用户任务：
+{task}
+
+候选 A：
+{answer_a}
+
+候选 B：
+{answer_b}
+"""
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("campaign", type=Path)
+    p.add_argument("--output", type=Path, required=True)
+    p.add_argument("--model", default="openai/gpt-oss-120b")
+    p.add_argument("--base-url", default="https://openrouter.ai/api/v1")
+    p.add_argument("--request-timeout", type=float, default=60.0)
+    return p.parse_args()
+
+
+def parse_judgment(text: str) -> dict[str, Any]:
+    winner = re.search(r"WINNER\s*:\s*(A|B|TIE)", text, re.I)
+    scores = [re.search(rf"SCORE_{name}\s*:\s*([0-4])", text, re.I) for name in ("A", "B")]
+    return {
+        "winner": winner.group(1).upper() if winner else None,
+        "score_a": int(scores[0].group(1)) if scores[0] else None,
+        "score_b": int(scores[1].group(1)) if scores[1] else None,
+        "parse_ok": bool(winner and all(scores)),
+    }
+
+
+def main() -> int:
+    args = parse_args()
+    api_key = os.getenv("OPENROUTER_API_KEY") or os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        raise SystemExit("quality judge requires OPENROUTER_API_KEY or OPENAI_API_KEY")
+    client = OpenAI(api_key=api_key, base_url=args.base_url, timeout=args.request_timeout)
+    campaign = json.loads(args.campaign.read_text(encoding="utf-8"))
+    tasks = {str(item["id"]): item["prompt"] for item in campaign["tasks"]}
+    by_pair: dict[str, dict[str, dict]] = {}
+    for run in campaign["runs"]:
+        by_pair.setdefault(str(run["pair_id"]), {})[run["path"]] = run
+    rng = random.Random(101)
+    pairs = []
+    for pair_id, arms in sorted(by_pair.items()):
+        if set(arms) != {"transfer", "skill"}:
+            continue
+        judgments = []
+        for repeat in range(2):
+            swapped = bool((rng.randrange(2) + repeat) % 2)
+            transfer = arms["transfer"]
+            skill = arms["skill"]
+            shown = [("skill", skill), ("transfer", transfer)] if swapped else [("transfer", transfer), ("skill", skill)]
+            prompt = JUDGE_PROMPT.format(
+                task=tasks[str(transfer["task_id"])],
+                answer_a=shown[0][1].get("final_answer", ""),
+                answer_b=shown[1][1].get("final_answer", ""),
+            )
+            kwargs = {
+                "model": args.model,
+                "messages": [{"role": "user", "content": prompt}],
+                "temperature": 0,
+                "max_tokens": 300,
+            }
+            started = datetime.now(timezone.utc).isoformat()
+            try:
+                response = client.chat.completions.create(**kwargs)
+            except Exception as exc:
+                if "temperature" not in str(exc).lower():
+                    raise
+                kwargs.pop("temperature", None)
+                response = client.chat.completions.create(**kwargs)
+            content = response.choices[0].message.content or ""
+            parsed = parse_judgment(content)
+            judgments.append({
+                "repeat": repeat + 1,
+                "shown_order": [shown[0][0], shown[1][0]],
+                "request": kwargs,
+                "response": response.model_dump(mode="json"),
+                "response_id": getattr(response, "id", None),
+                "captured_at": started,
+                "judgment": parsed,
+            })
+        # Convert each position-swapped judgment back to the architecture labels.
+        normalized = []
+        for item in judgments:
+            winner = item["judgment"]["winner"]
+            if winner == "TIE":
+                normalized.append("tie")
+            elif winner:
+                normalized.append(item["shown_order"][0 if winner == "A" else 1])
+            else:
+                normalized.append(None)
+        pairs.append({
+            "pair_id": pair_id,
+            "task_id": arms["transfer"]["task_id"],
+            "transfer_deterministic_pass": bool(arms["transfer"]["outcome"]["pass"]),
+            "skill_deterministic_pass": bool(arms["skill"]["outcome"]["pass"]),
+            "judgments": judgments,
+            "normalized_winners": normalized,
+            "parse_complete": all(item["judgment"]["parse_ok"] for item in judgments),
+        })
+    all_judgments = [item for pair in pairs for item in pair["judgments"]]
+    output = {
+        "schema_version": 1,
+        "experiment": "10-1",
+        "judge_model": args.model,
+        "judge_base_url": args.base_url,
+        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
+        "paired_n": len(pairs),
+        "position_swapped_repeats": 2,
+        "judge_receipt_count": len(all_judgments),
+        "unique_response_ids": len({item.get("response_id") for item in all_judgments}),
+        "parse_complete": all(item["parse_complete"] for item in pairs),
+        "pairs": pairs,
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(output, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(f"saved {args.output}: {len(pairs)} pairs, {len(all_judgments)} swapped judgments")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())