ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter3/user-memory-evaluation/comparison.py

@@ -0,0 +1,204 @@
+"""Cross-system comparison for the User Memory Evaluation Framework.
+
+Experiment 3-1 evaluates *memory systems*: the same three-layer test suite is run
+against several memory configurations (e.g. Simple Notes vs. Advanced JSON Cards)
+and their scores are compared side by side. This module takes a mapping of
+
+    {system_name: {test_id: agent_response}}
+
+scores every (system, test case) pair with the chosen metric, and renders a
+scored comparison table broken down by layer plus an overall row - so the reader
+can see, at a glance, which memory format wins on basic recall vs. cross-session
+synthesis.
+"""
+
+from typing import Dict, List, Optional
+
+from rich.console import Console
+from rich.table import Table
+
+from models import TestCase, EvaluationResult
+from metrics import KeywordRecallEvaluator
+
+console = Console()
+
+LAYERS = ["layer1", "layer2", "layer3"]
+LAYER_TITLES = {
+    "layer1": "Layer 1 · Basic Recall",
+    "layer2": "Layer 2 · Disambiguation",
+    "layer3": "Layer 3 · Proactive Synthesis",
+}
+
+
+class ComparisonRunner:
+    """Run one metric over several memory systems and compare their scores."""
+
+    def __init__(
+        self,
+        framework,
+        metric: str = "keyword-recall",
+        gold_facts: Optional[Dict] = None,
+        evaluator_type: Optional[str] = None,
+        model: Optional[str] = None,
+    ):
+        """
+        Args:
+            framework: A loaded UserMemoryEvaluationFramework (for test-case lookup).
+            metric: 'keyword-recall' (offline) or 'llm-judge' (needs API).
+            gold_facts: Gold-fact annotations required by the keyword-recall metric.
+            evaluator_type: Judge backend for 'llm-judge' (kimi/openai).
+            model: Optional model override for 'llm-judge'.
+        """
+        self.framework = framework
+        self.metric = metric
+        self.gold_facts = gold_facts or {}
+        self.evaluator_type = evaluator_type
+        self.model = model
+        self._evaluator = self._build_evaluator()
+
+    def _build_evaluator(self):
+        if self.metric == "keyword-recall":
+            return KeywordRecallEvaluator(self.gold_facts)
+        elif self.metric == "llm-judge":
+            # Imported lazily so the offline path never requires the openai client.
+            from evaluator import LLMEvaluator
+            return LLMEvaluator(self.evaluator_type, model=self.model)
+        raise ValueError(f"Unknown metric: {self.metric}. Supported: keyword-recall, llm-judge")
+
+    def run(
+        self,
+        system_responses: Dict[str, Dict[str, str]],
+        category: Optional[str] = None,
+    ) -> Dict[str, Dict[str, EvaluationResult]]:
+        """Score every system over the test cases it provides responses for.
+
+        Args:
+            system_responses: {system_name: {test_id: response}}.
+            category: Optional layer filter (layer1/layer2/layer3).
+
+        Returns:
+            {system_name: {test_id: EvaluationResult}}
+        """
+        results: Dict[str, Dict[str, EvaluationResult]] = {}
+        for system_name, responses in system_responses.items():
+            if system_name.startswith("_"):
+                continue  # skip JSON comment keys like "_comment"
+            system_results: Dict[str, EvaluationResult] = {}
+            for test_id, response in responses.items():
+                test_case = self.framework.get_test_case(test_id)
+                if not test_case:
+                    console.print(f"[yellow]Skipping unknown test case: {test_id}[/yellow]")
+                    continue
+                if category and test_case.category != category:
+                    continue
+                if self.metric == "keyword-recall" and not self._evaluator.has_gold(test_id):
+                    continue  # no gold facts -> not scorable offline
+                system_results[test_id] = self._evaluator.evaluate(test_case, response)
+            results[system_name] = system_results
+        return results
+
+    def _layer_of(self, test_id: str) -> str:
+        test_case = self.framework.get_test_case(test_id)
+        return test_case.category if test_case else "unknown"
+
+    def _avg(self, results: Dict[str, EvaluationResult], layer: Optional[str] = None) -> Optional[float]:
+        vals = [
+            r.reward
+            for tid, r in results.items()
+            if layer is None or self._layer_of(tid) == layer
+        ]
+        return sum(vals) / len(vals) if vals else None
+
+    def _count(self, results: Dict[str, EvaluationResult], layer: Optional[str] = None) -> int:
+        return sum(1 for tid in results if layer is None or self._layer_of(tid) == layer)
+
+    def build_table(self, results_by_system: Dict[str, Dict[str, EvaluationResult]]) -> Table:
+        """Build a Rich comparison table (layers as rows, systems as columns)."""
+        systems = list(results_by_system.keys())
+        metric_label = "Keyword Recall" if self.metric == "keyword-recall" else "LLM-as-Judge Reward"
+        table = Table(
+            title=f"Memory System Comparison ({metric_label}, 0.000-1.000)",
+            show_header=True,
+            header_style="bold cyan",
+        )
+        table.add_column("Layer", style="magenta", no_wrap=True)
+        for system in systems:
+            table.add_column(system, justify="center")
+
+        for layer in LAYERS:
+            row = [LAYER_TITLES[layer]]
+            has_any = False
+            for system in systems:
+                avg = self._avg(results_by_system[system], layer)
+                if avg is None:
+                    row.append("—")
+                else:
+                    has_any = True
+                    n = self._count(results_by_system[system], layer)
+                    row.append(f"{avg:.3f} (n={n})")
+            if has_any:
+                table.add_row(*row)
+
+        # Overall row
+        overall = ["[bold]Overall[/bold]"]
+        for system in systems:
+            avg = self._avg(results_by_system[system])
+            if avg is None:
+                overall.append("—")
+            else:
+                n = self._count(results_by_system[system])
+                overall.append(f"[bold]{avg:.3f} (n={n})[/bold]")
+        table.add_section()
+        table.add_row(*overall)
+        return table
+
+    def generate_report(self, results_by_system: Dict[str, Dict[str, EvaluationResult]]) -> str:
+        """Generate a plain-text comparison report (for saving to --output)."""
+        systems = list(results_by_system.keys())
+        metric_label = "keyword-recall" if self.metric == "keyword-recall" else "llm-judge"
+        lines = []
+        lines.append("=" * 80)
+        lines.append("MEMORY SYSTEM COMPARISON REPORT")
+        lines.append(f"Metric: {metric_label} (score range 0.000-1.000)")
+        lines.append("=" * 80)
+        lines.append("")
+
+        # Summary matrix
+        header = f"{'Layer':<32}" + "".join(f"{s:>18}" for s in systems)
+        lines.append(header)
+        lines.append("-" * len(header))
+        for layer in LAYERS:
+            cells = []
+            printed = False
+            for system in systems:
+                avg = self._avg(results_by_system[system], layer)
+                if avg is None:
+                    cells.append(f"{'—':>18}")
+                else:
+                    printed = True
+                    n = self._count(results_by_system[system], layer)
+                    cells.append(f"{avg:.3f} (n={n})".rjust(18))
+            if printed:
+                lines.append(f"{LAYER_TITLES[layer]:<32}" + "".join(cells))
+        overall_cells = []
+        for system in systems:
+            avg = self._avg(results_by_system[system])
+            overall_cells.append((f"{avg:.3f}" if avg is not None else "—").rjust(18))
+        lines.append("-" * len(header))
+        lines.append(f"{'Overall':<32}" + "".join(overall_cells))
+        lines.append("")
+
+        # Per-test-case detail
+        all_test_ids = sorted({tid for r in results_by_system.values() for tid in r})
+        lines.append("Per-test-case scores")
+        lines.append("-" * len(header))
+        lines.append(f"{'Test Case':<32}" + "".join(f"{s:>18}" for s in systems))
+        for tid in all_test_ids:
+            cells = []
+            for system in systems:
+                r = results_by_system[system].get(tid)
+                cells.append((f"{r.reward:.3f}" if r else "—").rjust(18))
+            lines.append(f"{tid:<32}" + "".join(cells))
+        lines.append("")
+
+        return "\n".join(lines)