#!/usr/bin/env python3 """把「AI 味」反馈开放式提炼为规则，并由 LLM judge 校准和评估。 默认使用 OpenAI GPT-5.6 Sol： python run_ai_style_skill.py 流程：全量反馈 → LLM 开放式提炼并做语义归并 → 每条规则用独立人工 金标校准 LLM judge → 生成 Skill → boundary/retention 评估 → 改写演示。 代码中不包含预置模式库，也不会用 detector 指纹过滤模型发现的新规律。 """ from __future__ import annotations import argparse from datetime import datetime, timezone import json from pathlib import Path import shutil from typing import Any, Dict, List from evaluate import evaluate_rules, load_eval_texts from extract_rules import extract_with_llm, load_pairs, write_candidates from judge import calibrate, llm_judge, load_golden_set from llm_client import default_model from rewrite_demo import SAMPLE_TEXT, rewrite_with_llm from skill_manager import merge_rules, prune_rules, write_archive, write_skill ROOT = Path(__file__).resolve().parent # 验收门槛（模型外部代码，LLM 不可修改）。 GATE_BOUNDARY_RATE = 0.85 GATE_RETENTION_HARM = 0.15 def run_pipeline( *, provider: str = "openai", model: str | None = None, seed: int = 8901, batches: int = 3, ) -> Dict[str, Any]: pairs = load_pairs() eval_texts = load_eval_texts() golden_set = load_golden_set() receipts: List[Dict[str, Any]] = [] # 1) 一次查看全部反馈，避免批次顺序让同一概念被拆分、不同概念被误并。 candidates, receipt = extract_with_llm( pairs, provider=provider, model=model, seed=seed, ) receipts.append(receipt) rules, merge_report = merge_rules([], candidates) pair_batches = {pair["id"]: pair.get("batch", 1) for pair in pairs} for rule in rules: rule["last_confirmed_batch"] = max( (pair_batches[source] for source in rule.get("source_ids", []) if source in pair_batches), default=batches, ) curve = [ { "batch": batch_no, "rules_with_evidence": sum( any(pair_batches.get(source) == batch_no for source in rule.get("source_ids", [])) for rule in rules ), "cumulative_rule_count": sum( any(pair_batches.get(source, batches + 1) <= batch_no for source in rule.get("source_ids", [])) for rule in rules ), } for batch_no in range(1, batches + 1) ] total_candidates = len(candidates) write_candidates(candidates) # 2) 所有规则都由同一个批量 LLM judge 校准；无金标或不达标都拒绝上线。 calibration = [] active: List[Dict[str, Any]] = [] judge_fn = llm_judge( provider=provider, model=model, seed=seed, receipts=receipts ) for rule in rules: result = calibrate(rule, golden_set, judge_fn) calibration.append(result) if result["decision"] == "activate": rule["status"] = "active" active.append(rule) else: rule["status"] = "rejected" # 3) prune 演示：被证据推翻或长期未触发的规则归档（离线数据下无归档）。 active, archived = prune_rules(active, current_batch=batches, contradicted_ids=set()) # 4) 生成 Skill 并评估。 skill_path = write_skill(active) archive_path = write_archive(archived) metrics = evaluate_rules(active, eval_texts, judge_fn) # 5) 改写演示。 rewrite, receipt = rewrite_with_llm( SAMPLE_TEXT, active, provider=provider, model=model, seed=seed ) receipts.append(receipt) gates = { "boundary_detection_rate >= 0.85": metrics["boundary_detection_rate"] >= GATE_BOUNDARY_RATE, "retention_harm_rate <= 0.15": metrics["retention_harm_rate"] <= GATE_RETENTION_HARM, "corpus_rules_have_unique_ids": ( len(rules) == total_candidates and len({rule["id"] for rule in rules}) == len(rules) and not merge_report["merged"] ), "all_active_rules_use_llm_judge": all( rule.get("detector") == {"type": "llm"} for rule in active ), "all_active_rules_calibrated": all( any(c["rule_id"] == rule["id"] and c["decision"] == "activate" for c in calibration) for rule in active ), "out_of_library_cases_detected": all( any(detail["expected"] and set(detail["expected"]) & set(detail["fired"]) for detail in metrics["boundary_details"] if detail["id"] == case_id) for case_id in ("b9", "b10", "b11") ), "real_llm_called_with_receipts": bool(receipts) and all( receipt["response"].get("id") for receipt in receipts ), } report = { "experiment": "ai-style-skill", "executed_at": datetime.now(timezone.utc).isoformat(), "execution_mode": "real_llm_judge", "provider": provider, "model": model or default_model(provider), "growth_curve": curve, "total_candidates": total_candidates, "final_rule_count": len(rules), "active_rules": [r["id"] for r in active], "archived_rules": [r["id"] for r in archived], "calibration": calibration, "skill_path": str(skill_path.relative_to(ROOT)), "archive_path": str(archive_path.relative_to(ROOT)) if archive_path else None, "metrics": metrics, "rewrite_demo": rewrite, "raw_api_receipts": receipts, "gates": gates, "accepted": all(gates.values()), } return report def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--provider", choices=("ark", "openrouter", "openai"), default="openai") parser.add_argument("--model", default=None) parser.add_argument("--seed", type=int, default=8901) args = parser.parse_args() report = run_pipeline(provider=args.provider, model=args.model, seed=args.seed) stamp = datetime.now(timezone.utc).strftime("real_%Y%m%dT%H%M%SZ") out_dir = ROOT / "validation" / stamp out_dir.mkdir(parents=True, exist_ok=False) evidence_path = out_dir / "evidence.json" evidence_path.write_text( json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8" ) canonical = ROOT / "validation" / "latest.json" shutil.copyfile(evidence_path, canonical) print(f"证据回执：{evidence_path.relative_to(ROOT)}（validation/latest.json 已指向）") metrics = report["metrics"] print(json.dumps({ "mode": report["execution_mode"], "accepted": report["accepted"], "boundary_detection_rate": f"{metrics['boundary_detected']}/{metrics['boundary_total']}", "retention_harm_rate": f"{metrics['retention_harmed']}/{metrics['retention_total']}", "candidates_to_rules": f"{report['total_candidates']} -> {report['final_rule_count']}", "active_rules": report["active_rules"], "calibration": [ {"rule_id": c["rule_id"], "agreement": c["agreement"], "decision": c["decision"]} for c in report["calibration"] ], "gates": report["gates"], }, ensure_ascii=False, indent=2)) return 0 if report["accepted"] else 1 if __name__ == "__main__": raise SystemExit(main())