ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter9/trajectory-verifier/llm_judge.py

@@ -0,0 +1,119 @@
+"""Real OpenAI Responses API judge for Experiment 9-1."""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Dict, Iterable
+
+try:
+    from dotenv import load_dotenv
+    load_dotenv()
+except ImportError:
+    pass
+
+from verifier import DimensionResult, FAIL, PASS, UNCERTAIN
+
+
+def _json_object(text: str) -> Dict[str, Any]:
+    cleaned = text.strip()
+    if cleaned.startswith("```"):
+        cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", cleaned, flags=re.IGNORECASE)
+    try:
+        return json.loads(cleaned)
+    except json.JSONDecodeError:
+        match = re.search(r"\{.*\}", cleaned, re.DOTALL)
+        if not match:
+            raise
+        return json.loads(match.group(0))
+
+
+class OpenAIQualityJudge:
+    """Evaluate open-ended quality while citing concrete dialogue turns."""
+
+    def __init__(self, model: str | None = None, *, evidence_client=None):
+        if evidence_client is None:
+            from evidence_client import EvidenceChatClient
+            evidence_client = EvidenceChatClient("openrouter", model)
+        self.client = evidence_client
+        self.model = evidence_client.model
+
+    def evaluate(self, trajectory: Dict[str, Any]) -> Iterable[DimensionResult]:
+        if not isinstance(trajectory, dict):
+            trajectory = {}
+        facts = trajectory.get("process_facts")
+        if not isinstance(facts, dict):
+            facts = {}
+        messages = trajectory.get("messages")
+        if not isinstance(messages, list):
+            messages = []
+        tool_calls = trajectory.get("tool_calls")
+        if not isinstance(tool_calls, list):
+            tool_calls = []
+        checked_rules = facts.get("checked_rules")
+        if not isinstance(checked_rules, list):
+            checked_rules = []
+        evidence = {
+            "user_request": trajectory.get("user_request"),
+            "messages": messages,
+            "tool_calls": tool_calls,
+            "checked_rules": checked_rules,
+        }
+        prompt = f"""You are calibrating a customer-service Agent trajectory.
+
+Evaluate exactly two dimensions and keep their scopes separate from the code-checked layers:
+1. expression_quality: ONLY whether wording is natural, concise and non-repetitive. Do not fail this dimension for factual, privacy, policy or action errors; those are checked elsewhere. Raw JSON presented to a customer is not natural expression.
+2. compliant_flexibility: if the requested business path is blocked, find an allowed alternative without breaking policy. The user's explicit fallback request is evidence of an available alternative. If no business path is blocked, return pass (not uncertain), because no workaround was needed. Do not use this dimension to re-score privacy.
+
+For each dimension return verdict (pass, fail, or uncertain), score from 0 to 1,
+confidence from 0 to 1, and an evidence array citing concrete turn numbers. If
+the transcript lacks enough evidence, use uncertain. Return JSON only:
+{{"dimensions": [{{"dimension": "expression_quality", "verdict": "pass", "score": 1.0, "confidence": 0.8, "evidence": ["turn 2: ..."]}}, ...]}}
+
+Trajectory evidence:
+{json.dumps(evidence, ensure_ascii=False, indent=2)}
+"""
+        response = self.client.complete(
+            kind="quality_judge",
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0,
+            response_format={"type": "json_object"},
+        )
+        payload = _json_object(response.choices[0].message.content or "{}")
+        if isinstance(payload, list):
+            raw_dims = payload
+        elif isinstance(payload, dict):
+            raw_dims = payload.get("dimensions")
+        else:
+            raw_dims = []
+        if not isinstance(raw_dims, list):
+            raw_dims = []
+        by_name = {
+            item.get("dimension"): item
+            for item in raw_dims
+            if isinstance(item, dict) and item.get("dimension")
+        }
+        results = []
+        for name in ("expression_quality", "compliant_flexibility"):
+            item = by_name.get(name) or {}
+            verdict = item.get("verdict", UNCERTAIN)
+            if verdict not in {PASS, FAIL, UNCERTAIN}:
+                verdict = UNCERTAIN
+            # dict.get(key, default) returns the default only when the key is
+            # ABSENT; a model that emits an explicit JSON null (common for a
+            # dimension it marks "uncertain") returns None, and float(None) /
+            # iterating None both raise. Coerce non-numeric / non-list values to
+            # the neutral defaults instead of crashing the whole trajectory.
+            score = item.get("score")
+            confidence = item.get("confidence")
+            evidence = item.get("evidence")
+            clean_evidence = [str(v) for v in evidence if v is not None] if isinstance(evidence, list) else []
+            results.append(DimensionResult(
+                dimension=name,
+                layer="llm_rubric",
+                verdict=verdict,
+                score=float(score) if isinstance(score, (int, float)) and not isinstance(score, bool) else 0.5,
+                evidence=clean_evidence if clean_evidence else ["LLM returned no evidence"],
+                confidence=float(confidence) if isinstance(confidence, (int, float)) and not isinstance(confidence, bool) else 0.5,
+            ))
+        return results