ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter9/prompt-auto-optimization/demo.py

@@ -0,0 +1,330 @@
+"""
+实验 9-3：基于失败轨迹的系统提示词自动优化
+
+一条命令跑通完整流程：
+  1. 用【初始 prompt】评测 → 暴露"政策争议就转人工"的过度转接问题；
+  2. 从失败轨迹生成三维诊断，保留来源案例；
+  3. Coding Agent 生成候选 prompt 的最小 diff；
+  4. 用边界集与保留集决定候选版本是否可灰度发布；
+  5. 与人工调优版对照。
+
+    python demo.py            # 完整运行：10 个用例 × 3 份 prompt
+    python demo.py --quick    # 快速演示：每组只取 2 个用例，省时省钱
+    python demo.py --help     # 查看全部命令行参数（中文说明）
+"""
+
+import argparse
+import json
+import os
+import shutil
+import sys
+import time
+from datetime import datetime, timezone
+
+from evaluate import evaluate_prompt
+from coding_agent import optimize_prompt
+from config import (
+    get_api_turns,
+    get_backend_metadata,
+    get_provider,
+    get_model,
+    reset_api_turns,
+    usage_summary,
+)
+from airline_env import CASES
+from learning_signal import diagnose_failures, format_learning_signal
+from release_gate import build_candidate_manifest, evaluate_release_gate
+
+GROUPS = ("holdout", "boundary")
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+INITIAL_PROMPT = os.path.join(HERE, "prompts", "system_prompt.txt")
+MANUAL_PROMPT = os.path.join(HERE, "prompts", "system_prompt_manual.txt")
+WORKING_PROMPT = os.path.join(HERE, "runtime", "system_prompt_working.txt")
+
+def _read(path):
+    with open(path, "r", encoding="utf-8") as f:
+        return f.read()
+
+
+def _pct(cn):
+    c, n = cn
+    return f"{c}/{n} ({100 * c / n:.0f}%)" if n else "-"
+
+
+def print_table(rows):
+    """rows: list of (label, holdout_tuple, boundary_tuple)"""
+    print("\n" + "=" * 74)
+    print("正确率对比（保留任务集 = 既有正确行为不能退化；边界案例集 = 过度转接应改善）")
+    print("=" * 74)
+    header = f"{'系统提示词版本':<26}{'保留任务集(holdout)':<20}{'边界案例集(boundary)':<20}"
+    print(header)
+    print("-" * 74)
+    for label, holdout, boundary in rows:
+        print(f"{label:<24}{_pct(holdout):<22}{_pct(boundary):<22}")
+    print("=" * 74)
+
+
+def _select_cases(limit_per_group=None, groups=GROUPS):
+    """按分组筛选用例，并对每组最多取 limit_per_group 个（None 表示不限制）。"""
+    picked, counts = [], {}
+    for c in CASES:
+        g = c["group"]
+        if g not in groups:
+            continue
+        if limit_per_group and counts.get(g, 0) >= limit_per_group:
+            continue
+        picked.append(c)
+        counts[g] = counts.get(g, 0) + 1
+    return picked
+
+
+def main(cases=None, rounds=3, output=None):
+    if cases is None:
+        cases = CASES
+    reset_api_turns()
+    campaign_started = time.time()
+    print("#" * 74)
+    print("# 实验 9-3：基于失败轨迹的系统提示词自动优化（航空客服场景）")
+    print(f"# LLM 提供商: {get_provider()}   模型: {get_model()}")
+    print(f"# 用例数: {len(cases)}（保留集 + 边界集）   Coding Agent 优化轮数上限: {rounds}")
+    print("#" * 74)
+
+    # ---- 准备：把初始 prompt 复制成本次运行的工作副本（Coding Agent 会改写它）----
+    os.makedirs(os.path.dirname(WORKING_PROMPT), exist_ok=True)
+    shutil.copyfile(INITIAL_PROMPT, WORKING_PROMPT)
+
+    # ---- 步骤 1：评测初始 prompt ----
+    print("\n【步骤 1】用初始系统提示词评测（观察是否过度转接）")
+    before = evaluate_prompt(_read(INITIAL_PROMPT), label="初始 prompt", cases=cases)
+    print(
+        f"\n  初始结果：保留集 {_pct(before['holdout'])}，"
+        f"边界集 {_pct(before['boundary'])}"
+    )
+    over_transfer = [
+        r for r in before["results"]
+        if r["group"] == "boundary" and not r["should_transfer"] and r["transferred"]
+    ]
+    print(f"  边界案例中出现【过度转接】的用例数：{len(over_transfer)} / "
+          f"{len([r for r in before['results'] if r['group'] == 'boundary'])}")
+    for r in over_transfer:
+        print(f"    - {r['id']}：政策争议却直接转人工，原因『{r['transfer_reason']}』")
+
+    # ---- 步骤 2：由失败轨迹形成学习信号 ----
+    learning_signal = diagnose_failures(before)
+    print("\n【步骤 2】将失败轨迹整理为三维诊断")
+    print(format_learning_signal(learning_signal))
+
+    # ---- 步骤 3：Coding Agent 生成候选 prompt ----
+    print("\n【步骤 3】Coding Agent 读取诊断并生成候选系统提示词……")
+    candidate_started = time.time()
+    opt = optimize_prompt(WORKING_PROMPT, learning_signal, max_rounds=rounds, verbose=True)
+    failure_to_candidate_seconds = time.time() - candidate_started
+    manifest = build_candidate_manifest(opt, learning_signal)
+    print(f"\n  Coding Agent 改动说明：{opt['rationale']}")
+    print("\n  ---------- 系统提示词文件 diff（真实写入磁盘）----------")
+    print(opt["diff"] if opt["diff"].strip() else "  (无改动)")
+    print("  --------------------------------------------------------")
+
+    print(f"  候选补丁来源：{', '.join(manifest['source_case_ids'])}")
+    print(f"  候选补丁作用域：{manifest['scope']}")
+
+    # ---- 步骤 4：评测候选 prompt 并运行发布门槛 ----
+    print("\n【步骤 4】评测候选系统提示词并运行发布门槛")
+    after = evaluate_prompt(opt["after"], label="自动优化后 prompt", cases=cases)
+    gate = evaluate_release_gate(before, after, manifest)
+    print(f"  发布决定：{gate['decision']}")
+    for check, passed in gate["checks"].items():
+        print(f"    {'✓' if passed else '✗'} {check}")
+
+    # ---- 步骤 5：对照人工调优版 ----
+    print("\n【步骤 5】对照组：人工调优版系统提示词")
+    manual = evaluate_prompt(_read(MANUAL_PROMPT), label="人工调优版 prompt(对照)", cases=cases)
+
+    # ---- 步骤 6：对比表 ----
+    print_table([
+        ("初始 prompt(优化前)", before["holdout"], before["boundary"]),
+        ("自动优化后 prompt", after["holdout"], after["boundary"]),
+        ("人工调优版(对照)", manual["holdout"], manual["boundary"]),
+    ])
+
+    # ---- 结论 ----
+    b_before_c, b_before_n = before["boundary"]
+    b_after_c, _ = after["boundary"]
+    h_before_c, _ = before["holdout"]
+    h_after_c, _ = after["holdout"]
+    print("\n【结论】")
+    print(f"  · 边界案例集正确率：{b_before_c}/{b_before_n} → {b_after_c}/{b_before_n} "
+          f"（{'提升 ✓' if b_after_c > b_before_c else '未提升'}）")
+    print(f"  · 保留任务集正确率：{h_before_c} → {h_after_c} "
+          f"（{'未退化 ✓' if h_after_c >= h_before_c else '退化 ✗'}）")
+    print(f"\n  候选工作副本已写入：{WORKING_PROMPT}")
+    print("  它不会覆盖稳定版本；只有 release_to_canary 才允许进入灰度。")
+
+    # ---- 可选：把对比结果落盘为 JSON，便于复现与二次分析 ----
+    before_by_id = {row["id"]: row for row in before["results"]}
+    after_by_id = {row["id"]: row for row in after["results"]}
+    regressions = [
+        identifier for identifier, old in before_by_id.items()
+        if old["correct"] and not after_by_id[identifier]["correct"]
+    ]
+    boundary_fixed = [
+        identifier for identifier, old in before_by_id.items()
+        if old["group"] == "boundary" and not old["correct"] and after_by_id[identifier]["correct"]
+    ]
+    api_turns = get_api_turns()
+    gates = [
+        {"name": "full_holdout_and_boundary_sets_run", "passed": len(cases) == len(CASES) and {c["group"] for c in cases} == {"holdout", "boundary"}, "evidence": {"selected": len(cases), "canonical": len(CASES)}},
+        {"name": "same_model_and_same_cases_for_three_controls", "passed": all({r["id"] for r in report["results"]} == {c["id"] for c in cases} for report in (before, after, manual)), "evidence": get_model()},
+        {"name": "real_task_agent_calls", "passed": any(turn["kind"].startswith("task_agent") for turn in api_turns), "evidence": sum(turn["kind"].startswith("task_agent") for turn in api_turns)},
+        {"name": "real_llm_judge_calls", "passed": any(turn["kind"] == "llm_judge" for turn in api_turns), "evidence": sum(turn["kind"] == "llm_judge" for turn in api_turns)},
+        {"name": "real_coding_agent_call", "passed": any(turn["kind"] == "coding_agent" for turn in api_turns), "evidence": sum(turn["kind"] == "coding_agent" for turn in api_turns)},
+        {"name": "learning_signal_has_three_dimensions_and_source_ids", "passed": set(learning_signal["dimensions"]) == {"rule_compliance", "task_resolution", "compliant_flexibility"} and bool(learning_signal["source_case_ids"]), "evidence": learning_signal["source_case_ids"]},
+        {"name": "minimal_old_to_new_patch_is_auditable", "passed": bool(manifest.get("edits")) and bool(manifest.get("diff")), "evidence": manifest.get("edits")},
+        {"name": "release_gate_evaluated_all_four_manuscript_conditions", "passed": set(gate["checks"]) >= {"patch_is_nonempty", "patch_is_auditable_old_to_new_edit", "source_cases_are_recorded", "holdout_did_not_regress", "boundary_improved"}, "evidence": gate["checks"]},
+        {"name": "stable_prompt_not_overwritten", "passed": _read(INITIAL_PROMPT) == opt["before"], "evidence": {"stable": INITIAL_PROMPT, "candidate": WORKING_PROMPT}},
+        {"name": "raw_credential_free_api_receipts_saved", "passed": bool(api_turns), "evidence": len(api_turns)},
+    ]
+    execution_accepted = all(item["passed"] for item in gates)
+    result_claims = {
+        "boundary_improved": after["boundary"][0] > before["boundary"][0],
+        "holdout_not_degraded": after["holdout"][0] >= before["holdout"][0],
+        "automatic_candidate_released_only_to_canary": gate["decision"] == "release_to_canary",
+        "automatic_candidate_compared_with_manual": True,
+    }
+    summary = {
+            "schema_version": 2,
+            "experiment_id": "9-3",
+            "canonical_source": "book/chapter9.md#实验-9-3-基于失败轨迹优化系统提示词",
+            "evidence_mode": "real_task_agent_llm_judge_coding_agent_full_campaign",
+            "created_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
+            "provider": get_provider(),
+            "model": get_model(),
+            "backend": get_backend_metadata(),
+            "credential_value_recorded": False,
+            "rounds": rounds,
+            "num_cases": len(cases),
+            "case_ids": [case["id"] for case in cases],
+            "learning_signal": learning_signal,
+            "candidate_manifest": manifest,
+            "release_gate": gate,
+            "rationale": opt["rationale"],
+            "diff": opt["diff"],
+            "prompt_metrics": {
+                "initial_characters": len(opt["before"]),
+                "candidate_characters": len(opt["after"]),
+                "growth_characters": len(opt["after"]) - len(opt["before"]),
+                "manual_characters": len(_read(MANUAL_PROMPT)),
+                "introduced_regressions": len(regressions),
+                "regression_case_ids": regressions,
+                "boundary_failures_fixed": len(boundary_fixed),
+                "boundary_fixed_case_ids": boundary_fixed,
+                "failure_to_candidate_seconds": round(failure_to_candidate_seconds, 6),
+                "campaign_elapsed_seconds": round(time.time() - campaign_started, 6),
+            },
+            "evaluations": {"initial": before, "automatic_candidate": after, "manual": manual},
+            "rows": [
+                {"label": "初始 prompt(优化前)", "holdout": list(before["holdout"]),
+                 "boundary": list(before["boundary"])},
+                {"label": "自动优化后 prompt", "holdout": list(after["holdout"]),
+                 "boundary": list(after["boundary"])},
+                {"label": "人工调优版(对照)", "holdout": list(manual["holdout"]),
+                 "boundary": list(manual["boundary"])},
+            ],
+            "usage": usage_summary(),
+            "api_turns": api_turns,
+            "acceptance": {
+                "gates": gates,
+                "execution_accepted": execution_accepted,
+                "result_claims": result_claims,
+                "all_manuscript_result_claims_observed": all(result_claims.values()),
+            },
+        }
+    if output:
+        os.makedirs(os.path.dirname(os.path.abspath(output)), exist_ok=True)
+        with open(output, "w", encoding="utf-8") as f:
+            json.dump(summary, f, ensure_ascii=False, indent=2)
+        print(f"  对比结果已写入：{output}")
+    return summary
+
+
+def _build_parser():
+    parser = argparse.ArgumentParser(
+        prog="demo.py",
+        description="实验 9-3：从失败轨迹诊断到候选补丁与发布门槛（航空客服场景）。",
+        formatter_class=argparse.RawTextHelpFormatter,
+        epilog=(
+            "示例：\n"
+            "  python demo.py                     # 完整运行：10 个用例 × 3 份 prompt\n"
+            "  python demo.py --quick             # 每组只取 2 个用例，省时省钱\n"
+            "  python demo.py --group boundary    # 只评测边界案例集\n"
+            "  python demo.py --rounds 5 --model gpt-5.6-luna\n"
+            "  python demo.py --output output/run.json  # 把对比结果写成 JSON\n"
+            "  python demo.py --dry-run           # 离线：只打印配置与用例数，不调用 API"
+        ),
+    )
+    parser.add_argument(
+        "--quick", action="store_true",
+        help="快速演示模式：每组只取 2 个用例，减少 API 调用与耗时。",
+    )
+    parser.add_argument(
+        "--limit", type=int, default=None, metavar="N",
+        help="每组最多评测 N 个用例（覆盖 --quick）。",
+    )
+    parser.add_argument(
+        "--group", choices=("holdout", "boundary", "both"), default="both",
+        help="选择评测的任务集：holdout(保留集) / boundary(边界集) / both(默认，两者都跑)。",
+    )
+    parser.add_argument(
+        "--rounds", type=int, default=3, metavar="N",
+        help="Coding Agent 自动改写提示词的最大重试轮数（默认 3）。",
+    )
+    parser.add_argument(
+        "--model", default=None, metavar="NAME",
+        help="覆盖 LLM 模型名（等价于设置环境变量 LLM_MODEL，如 gpt-5.6-luna）。",
+    )
+    parser.add_argument(
+        "--provider", choices=("openai", "moonshot", "ark", "openrouter"), default=None,
+        help="覆盖 LLM 提供商（等价于设置环境变量 LLM_PROVIDER，默认 openai）。",
+    )
+    parser.add_argument(
+        "--output", default=None, metavar="PATH",
+        help="把优化前后 + 人工对照的对比结果写入指定 JSON 文件（如 output/run.json）。",
+    )
+    parser.add_argument(
+        "--dry-run", action="store_true",
+        help="离线自检：只打印解析后的配置与选中用例数，不调用任何 LLM API。",
+    )
+    return parser
+
+
+if __name__ == "__main__":
+    args = _build_parser().parse_args()
+
+    # 命令行覆盖优先级高于环境变量：get_provider()/get_model() 均在调用时读取环境变量
+    if args.provider:
+        os.environ["LLM_PROVIDER"] = args.provider
+    if args.model:
+        os.environ["LLM_MODEL"] = args.model
+
+    limit = args.limit if args.limit is not None else (2 if args.quick else None)
+    groups = GROUPS if args.group == "both" else (args.group,)
+    cases = _select_cases(limit, groups=groups)
+
+    if args.dry_run:
+        # 离线路径：不触发任何网络请求，仅用于验证参数解析与用例选择
+        print("[dry-run] 解析后的运行配置（不调用 API）：")
+        print(f"  LLM 提供商 : {get_provider()}")
+        print(f"  LLM 模型   : {get_model()}")
+        print(f"  优化轮数   : {args.rounds}")
+        print(f"  任务集     : {args.group}")
+        print(f"  选中用例数 : {len(cases)}  -> {[c['id'] for c in cases]}")
+        print(f"  输出文件   : {args.output or '(不写文件)'}")
+        sys.exit(0)
+
+    try:
+        main(cases=cases, rounds=args.rounds, output=args.output)
+    except RuntimeError as e:
+        # 例如 API Key 未设置：给出清晰的人类可读错误，而非原始 traceback
+        print(f"\n[错误] {e}", file=sys.stderr)
+        sys.exit(1)