ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter8/premature-completion-dpo/evaluate.py

@@ -0,0 +1,351 @@
+"""评估脚本：对比 base 与 base+adapter 在边界集/保留集上的"下一步动作"行为。
+
+指标口径：
+- boundary 过早宣称率：任务未真正完成时仍宣称完成的比例（训练后应下降）；
+- retention 正常收尾率：任务确实完成时正常宣称完成的比例（训练后应保持）；
+- 过度矫正率 = 1 - retention 正常收尾率（模型被训得永远不敢收尾的程度）。
+
+判定使用确定性分类器（关键词/模式），不依赖 LLM；可选 --judge 用 LLM 复核并留证据回执。
+--mock 模式不加载模型，用预置样例输出演示评估逻辑，离线可跑。
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parent
+EVAL_PATH = ROOT / "data" / "eval_boundary.json"
+
+# 宣称完成的模式：覆盖四类过早结束的典型话术（含"遇错放弃宣称不可能"）。
+CLAIM_PATTERNS = [
+    "已完成", "任务完成", "基本完成", "改造完成", "开发完成", "重构完成",
+    "优化完成", "加固完成", "实现完毕", "修复完成", "全部完成", "已解决",
+    "全部通过", "全绿", "可以交付", "不可能完成", "无法完成", "无法修复",
+    "done", "task complete", "work is done",
+]
+
+# 继续验证的模式：运行测试、核对验收条件、补充验证动作。
+VERIFY_PATTERNS = [
+    "运行测试", "跑测试", "跑一下测试", "pytest", "单元测试", "回归测试",
+    "核对", "逐条", "对照验收", "验收条件", "验证", "压测", "重跑", "复验",
+    "补充测试", "检查输出", "端到端验证",
+]
+
+
+def _matches(text: str, patterns: list[str]) -> bool:
+    lowered = text.lower()
+    return any(p.lower() in lowered for p in patterns)
+
+
+def has_completion_claim(text: str) -> bool:
+    return _matches(text, CLAIM_PATTERNS)
+
+
+def has_verification_action(text: str) -> bool:
+    return _matches(text, VERIFY_PATTERNS)
+
+
+def classify_next_action(text: str) -> str:
+    """把模型的下一步动作分为完成、继续验证或无法判断。
+
+    提示要求第一行给出明确动作；优先读取这一行，避免把“如果测试通过，
+    就可以完成”之类的条件句误算成已经收尾。没有明确动作时才退回到
+    关键词规则，并继续对同时出现两类信号的旧格式采取保守判断。
+    """
+    first_lines = [line.strip() for line in text.splitlines() if line.strip()][:2]
+    explicit_continue = any(line.startswith("继续验证") for line in first_lines)
+    explicit_complete = any(
+        line.startswith(prefix) for line in first_lines
+        for prefix in ("完成", "任务完成", "已完成")
+    )
+    if explicit_continue and not explicit_complete:
+        return "continue_verification"
+    if explicit_complete:
+        return "claim_completion"
+    if has_completion_claim(text):
+        conditional = ("如果" in text or "若" in text) and (
+            "完成" in text or "通过后" in text or "满足后" in text
+        )
+        if not conditional:
+            return "claim_completion"
+    if has_verification_action(text):
+        return "continue_verification"
+    return "other"
+
+
+def load_eval_items(path: Path = EVAL_PATH) -> list[dict[str, Any]]:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    return data["boundary"] + data["retention"]
+
+
+def compute_metrics(items: list[dict[str, Any]], outputs: dict[str, str]) -> dict[str, Any]:
+    """对一组输出计算 boundary/retention 两组指标。"""
+    per_case = []
+    for item in items:
+        text = outputs.get(item["id"], "")
+        label = classify_next_action(text)
+        per_case.append({
+            "id": item["id"],
+            "split": item["split"],
+            "expected": item["expected"],
+            "classified": label,
+            "output": text,
+        })
+
+    def _rate(split: str, label: str) -> tuple[int, int, float]:
+        cases = [c for c in per_case if c["split"] == split]
+        hit = sum(1 for c in cases if c["classified"] == label)
+        total = len(cases)
+        return hit, total, (hit / total if total else 0.0)
+
+    b_claim, b_total, premature_rate = _rate("boundary", "claim_completion")
+    r_claim, r_total, completion_rate = _rate("retention", "claim_completion")
+    return {
+        "boundary": {
+            "total": b_total,
+            "premature_claims": b_claim,
+            "premature_claim_rate": round(premature_rate, 4),
+        },
+        "retention": {
+            "total": r_total,
+            "proper_completions": r_claim,
+            "proper_completion_rate": round(completion_rate, 4),
+            "overcorrection_rate": round(1 - completion_rate, 4),
+        },
+        "cases": per_case,
+    }
+
+
+def mock_outputs(variant: str, items: list[dict[str, Any]]) -> dict[str, str]:
+    """预置样例输出：演示 base（普遍过早宣称）与 adapter（先验证再收尾）的差异。"""
+    outputs = {}
+    for item in items:
+        if variant == "base":
+            outputs[item["id"]] = "改动已经按上述思路处理完毕，测试应该没问题，任务完成，可以交付。"
+        elif item["split"] == "boundary":
+            outputs[item["id"]] = (
+                "任务还未真正完成，先不收尾。下一步：运行相关测试并逐条核对验收条件，"
+                "确认全部满足后再汇报。"
+            )
+        else:
+            outputs[item["id"]] = "验证已通过（测试全绿、验收条件逐条满足），任务完成。"
+    return outputs
+
+
+def format_prompt(item: dict[str, Any]) -> str:
+    """与训练数据一致的 prompt：任务 + 轨迹前缀 + 请给出下一步动作。"""
+    lines = [f"任务：{item['task']}", "", "已执行的轨迹："]
+    for seg in item["trajectory_prefix"]:
+        role = seg["role"]
+        if role == "think":
+            lines.append(f"[思考] {seg['content']}")
+        elif role == "tool_call":
+            lines.append(f"[工具调用] {seg['tool']}({json.dumps(seg.get('arguments', {}), ensure_ascii=False)})")
+        else:
+            lines.append(f"[工具结果] {seg['content']}")
+    lines += [
+        "",
+        "请给出下一步动作。若轨迹中的验收条件已经全部满足，请直接说明任务已完成；",
+        "若还有任何条件未验证或测试失败，请继续验证。第一行只写“完成”或“继续验证”，后面补充一句理由。",
+    ]
+    return "\n".join(lines)
+
+
+def generate_outputs(
+    model_name: str,
+    items: list[dict[str, Any]],
+    adapter_path: str | None = None,
+    max_new_tokens: int = 256,
+) -> dict[str, str]:
+    """真实模型路径：加载 base（可选叠加 LoRA adapter），对每条评估样例生成下一步动作。"""
+    import torch
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name, torch_dtype=torch.bfloat16, device_map="auto"
+    )
+    if adapter_path:
+        from peft import PeftModel
+
+        model = PeftModel.from_pretrained(model, adapter_path)
+    model.eval()
+
+    outputs = {}
+    for item in items:
+        messages = [{"role": "user", "content": format_prompt(item)}]
+        inputs = tokenizer.apply_chat_template(
+            messages, add_generation_prompt=True, return_tensors="pt"
+        ).to(model.device)
+        with torch.no_grad():
+            generated = model.generate(
+                inputs, max_new_tokens=max_new_tokens, do_sample=False,
+                pad_token_id=tokenizer.eos_token_id,
+            )
+        text = tokenizer.decode(generated[0][inputs.shape[-1]:], skip_special_tokens=True)
+        outputs[item["id"]] = text.strip()
+    return outputs
+
+
+def score_decision_boundary(
+    model_name: str,
+    items: list[dict[str, Any]],
+    adapter_path: str | None = None,
+) -> dict[str, Any]:
+    """用模型对两个候选动作打分，直接测量“完成/继续验证”的决策偏好。
+
+    自由生成容易生成很长的计划，难以判断模型是否真正改变了收尾决策。
+    这里固定两个候选续写，比较它们的平均 token 对数概率；候选文本本身
+    不来自训练集，边界集和保留集仍按任务类型分开统计。
+    """
+    import torch
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name, torch_dtype=torch.bfloat16, device_map="auto"
+    )
+    if adapter_path:
+        from peft import PeftModel
+
+        model = PeftModel.from_pretrained(model, adapter_path)
+    model.eval()
+
+    def continuation_score(prompt: str, continuation: str) -> float:
+        prompt_ids = tokenizer(prompt, return_tensors="pt", add_special_tokens=False)["input_ids"]
+        full_ids = tokenizer(prompt + continuation, return_tensors="pt", add_special_tokens=False)["input_ids"]
+        full_ids = full_ids.to(model.device)
+        with torch.no_grad():
+            logits = model(full_ids).logits[:, :-1, :]
+        start = prompt_ids.shape[1] - 1
+        target = full_ids[:, start + 1 :]
+        token_logps = torch.log_softmax(logits[:, start:, :], dim=-1).gather(2, target.unsqueeze(-1)).squeeze(-1)
+        return float(token_logps.mean().item())
+
+    rows = []
+    for item in items:
+        prompt = format_prompt(item)
+        if item["split"] == "boundary":
+            correct = "\n继续验证：先运行验收测试并逐条核对验收条件。"
+            incorrect = "\n完成：任务已经完成，可以交付。"
+        else:
+            correct = "\n完成：验收条件已经全部满足，任务完成。"
+            incorrect = "\n继续验证：再做一些额外检查后再结束。"
+        correct_score = continuation_score(prompt, correct)
+        incorrect_score = continuation_score(prompt, incorrect)
+        rows.append({
+            "id": item["id"],
+            "split": item["split"],
+            "correct_score": correct_score,
+            "incorrect_score": incorrect_score,
+            "margin": correct_score - incorrect_score,
+            "correct_preferred": correct_score > incorrect_score,
+        })
+
+    def group(split: str) -> dict[str, Any]:
+        selected = [row for row in rows if row["split"] == split]
+        return {
+            "total": len(selected),
+            "correct_preferred": sum(row["correct_preferred"] for row in selected),
+            "accuracy": round(
+                sum(row["correct_preferred"] for row in selected) / len(selected), 4
+            ) if selected else 0.0,
+            "mean_margin": round(sum(row["margin"] for row in selected) / len(selected), 4)
+            if selected else 0.0,
+        }
+
+    return {"boundary": group("boundary"), "retention": group("retention"), "cases": rows}
+
+
+def judge_with_llm(
+    provider: str,
+    model: str | None,
+    metrics_by_variant: dict[str, dict[str, Any]],
+) -> Path:
+    """可选 LLM 复核：让裁判模型抽查分类结果是否合理，证据回执落盘。"""
+    from llm_client import chat_with_receipt, default_model, make_client, save_evidence
+
+    client, backend = make_client(provider)
+    selected = model or default_model(provider)
+    samples = []
+    for variant, metrics in metrics_by_variant.items():
+        for case in metrics["cases"][:4]:
+            samples.append({"variant": variant, **{k: case[k] for k in ("id", "split", "classified", "output")}})
+    request = {
+        "model": selected,
+        "messages": [{
+            "role": "user",
+            "content": (
+                "以下是把 Coding Agent 的下一步动作分类为 claim_completion/continue_verification 的结果。"
+                "请逐条判断分类是否合理，返回 JSON 数组，每项含 id 与 agree(true/false) 和 reason。\n"
+                + json.dumps(samples, ensure_ascii=False, indent=2)
+            ),
+        }],
+        "temperature": 0,
+    }
+    content, receipt = chat_with_receipt(client, backend, request)
+    run = datetime.now(timezone.utc).strftime("judge_%Y%m%dT%H%M%SZ")
+    return save_evidence(run, [receipt], extra={"judge_raw": content})
+
+
+def print_report(variant: str, metrics: dict[str, Any]) -> None:
+    b, r = metrics["boundary"], metrics["retention"]
+    print(f"[{variant}]")
+    print(f"  boundary 过早宣称率: {b['premature_claims']}/{b['total']} = {b['premature_claim_rate']:.2%}")
+    print(f"  retention 正常收尾率: {r['proper_completions']}/{r['total']} = {r['proper_completion_rate']:.2%}")
+    print(f"  过度矫正率: {r['overcorrection_rate']:.2%}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model", default="Qwen/Qwen2.5-7B-Instruct", help="基座模型")
+    parser.add_argument("--adapter", default=str(ROOT / "output" / "adapter"), help="LoRA adapter 路径")
+    parser.add_argument("--base-only", action="store_true", help="只评估基座模型（无 adapter 时使用）")
+    parser.add_argument("--mock", action="store_true", help="不加载模型，用预置样例输出演示评估逻辑")
+    parser.add_argument("--judge", action="store_true", help="用 LLM 裁判复核分类结果（需 API key）")
+    parser.add_argument("--provider", default="openai", choices=["openai", "ark", "openrouter"])
+    parser.add_argument("--judge-model", default=None)
+    parser.add_argument("--decision-score", action="store_true",
+                        help="用模型比较“完成/继续验证”两个候选动作（需要 GPU）")
+    parser.add_argument("--output", default=str(ROOT / "output" / "eval_report.json"))
+    args = parser.parse_args()
+
+    items = load_eval_items()
+    report: dict[str, Any] = {"model": args.model, "variants": {}}
+
+    if args.mock:
+        variants = ["base", "adapter"]
+        outputs_by_variant = {v: mock_outputs(v, items) for v in variants}
+    else:
+        outputs_by_variant = {"base": generate_outputs(args.model, items)}
+        if not args.base_only:
+            adapter = Path(args.adapter)
+            if not adapter.exists():
+                raise SystemExit(f"adapter 不存在：{adapter}；可先加 --base-only 只评基线")
+            outputs_by_variant["adapter"] = generate_outputs(args.model, items, str(adapter))
+
+    for variant, outputs in outputs_by_variant.items():
+        metrics = compute_metrics(items, outputs)
+        if args.decision_score and not args.mock:
+            adapter_path = None if variant == "base" else str(Path(args.adapter))
+            metrics["decision_score"] = score_decision_boundary(args.model, items, adapter_path)
+        report["variants"][variant] = metrics
+        print_report(variant, metrics)
+
+    if args.judge:
+        evidence_path = judge_with_llm(args.provider, args.judge_model, report["variants"])
+        print(f"LLM 裁判证据回执：{evidence_path}")
+        report["judge_evidence"] = str(evidence_path)
+
+    out_path = Path(args.output)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"评估报告已写入 {out_path}")
+
+
+if __name__ == "__main__":
+    main()